mirror of
https://github.com/modelstudioai/cli.git
synced 2026-09-14 19:49:23 +08:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 故璃
|
1c76749ee5 |
@@ -28,7 +28,7 @@ const UPLOAD_FLAGS = {
|
||||
type: "string",
|
||||
valueHint: "<s>",
|
||||
description:
|
||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
|
||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
|
||||
},
|
||||
noValidate: {
|
||||
type: "switch",
|
||||
@@ -44,7 +44,7 @@ export default defineCommand({
|
||||
description: "Upload a dataset file (.jsonl or .zip) to Bailian",
|
||||
auth: "apiKey",
|
||||
usageArgs:
|
||||
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image>] [--no-validate] [--full-validate]",
|
||||
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
|
||||
flags: UPLOAD_FLAGS,
|
||||
exampleArgs: [
|
||||
"--file train.jsonl",
|
||||
@@ -72,15 +72,8 @@ export default defineCommand({
|
||||
const filePath = flags.file;
|
||||
const purpose = flags.purpose || "fine-tune";
|
||||
const schema = parseDatasetSchemaFlag(flags.schema);
|
||||
if (schema === "video") {
|
||||
throw new BailianError(
|
||||
`--schema video is not supported.`,
|
||||
ExitCode.USAGE,
|
||||
`Supported schemas: chatml, dpo, cpt, tts, image.`,
|
||||
);
|
||||
}
|
||||
// Image schema allows larger ZIPs (1 GB vs 300 MB for text).
|
||||
const isMediaSchema = schema === "image";
|
||||
// Image and video schemas allow larger ZIPs (1 GB vs 300 MB for text).
|
||||
const isMediaSchema = schema === "image" || schema === "video";
|
||||
|
||||
if (!flags.noValidate) {
|
||||
const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;
|
||||
|
||||
@@ -23,7 +23,7 @@ const VALIDATE_FLAGS = {
|
||||
type: "string",
|
||||
valueHint: "<s>",
|
||||
description:
|
||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
|
||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
|
||||
},
|
||||
} satisfies FlagsDef;
|
||||
|
||||
@@ -31,13 +31,14 @@ export default defineCommand({
|
||||
description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
|
||||
// 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。
|
||||
auth: "none",
|
||||
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image>]",
|
||||
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
|
||||
flags: VALIDATE_FLAGS,
|
||||
exampleArgs: [
|
||||
"--file train.jsonl",
|
||||
"--file dpo.jsonl --schema dpo",
|
||||
"--file cpt.jsonl --schema cpt",
|
||||
"--file audio.zip --schema tts",
|
||||
"--file wan-i2v-training-dataset.zip --schema video",
|
||||
"--file eval.jsonl --full-validate",
|
||||
"--file train.jsonl --output json",
|
||||
],
|
||||
@@ -47,25 +48,20 @@ export default defineCommand({
|
||||
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
|
||||
'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
|
||||
'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
|
||||
'image = {img_path:"..."} (image generation). With no --schema, a record',
|
||||
"carrying wav_fn is validated as TTS, img_path as image, chosen/rejected",
|
||||
"as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to",
|
||||
"require a specific shape on every record. ZIP archives (.zip) are",
|
||||
"validated structurally (data.jsonl present, media references resolve) in",
|
||||
"addition to per-record content checks. Use --full-validate to JSON.parse",
|
||||
"every line.",
|
||||
'image = {img_path:"..."} (image generation);',
|
||||
'video = {first_frame_path:"...", video_path:"..."} (video generation,',
|
||||
"i2v first-frame or kf2v first+last-frame with last_frame_path). With no",
|
||||
"--schema, a record carrying wav_fn is validated as TTS, img_path as image,",
|
||||
"first_frame_path/video_path as video, chosen/rejected as DPO, text (no",
|
||||
"messages) as CPT, otherwise ChatML. Pass --schema to require a specific",
|
||||
"shape on every record. ZIP archives (.zip) are validated structurally",
|
||||
"(data.jsonl present, media references resolve) in addition to per-record",
|
||||
"content checks. Use --full-validate to JSON.parse every line.",
|
||||
],
|
||||
async run(ctx) {
|
||||
const { settings, flags } = ctx;
|
||||
const filePath = flags.file;
|
||||
const schema = parseDatasetSchemaFlag(flags.schema);
|
||||
if (schema === "video") {
|
||||
throw new BailianError(
|
||||
`--schema video is not supported.`,
|
||||
ExitCode.USAGE,
|
||||
`Supported schemas: chatml, dpo, cpt, tts, image.`,
|
||||
);
|
||||
}
|
||||
if (settings.dryRun) {
|
||||
emitResult(
|
||||
{
|
||||
|
||||
@@ -107,23 +107,23 @@ bl dataset list --output json
|
||||
|
||||
### `bl dataset upload`
|
||||
|
||||
| Field | Value |
|
||||
| --------------- | -------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Name** | `dataset upload` |
|
||||
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
|
||||
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image>] [--no-validate] [--full-validate]` |
|
||||
| Field | Value |
|
||||
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Name** | `dataset upload` |
|
||||
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
|
||||
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
|
||||
|
||||
#### Flags
|
||||
|
||||
| Flag | Type | Required | Description |
|
||||
| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
|
||||
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
|
||||
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
|
||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||
| `--api-key <key>` | string | no | API key |
|
||||
| `--base-url <url>` | string | no | API base URL |
|
||||
| Flag | Type | Required | Description |
|
||||
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
|
||||
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
|
||||
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
|
||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||
| `--api-key <key>` | string | no | API key |
|
||||
| `--base-url <url>` | string | no | API base URL |
|
||||
|
||||
#### Notes
|
||||
|
||||
@@ -170,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate
|
||||
|
||||
### `bl dataset validate`
|
||||
|
||||
| Field | Value |
|
||||
| --------------- | ----------------------------------------------------------------------------------------------- |
|
||||
| **Name** | `dataset validate` |
|
||||
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
|
||||
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image>]` |
|
||||
| Field | Value |
|
||||
| --------------- | ------------------------------------------------------------------------------------------------------ |
|
||||
| **Name** | `dataset validate` |
|
||||
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
|
||||
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
|
||||
|
||||
#### Flags
|
||||
|
||||
| Flag | Type | Required | Description |
|
||||
| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
|
||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
|
||||
| Flag | Type | Required | Description |
|
||||
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
|
||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
|
||||
|
||||
#### Notes
|
||||
|
||||
@@ -191,13 +191,15 @@ bl dataset upload --file train.jsonl --no-validate
|
||||
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
|
||||
- rejected}; cpt = {text:"..."} (continual pre-training, raw text);
|
||||
- tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
|
||||
- image = {img_path:"..."} (image generation). With no --schema, a record
|
||||
- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected
|
||||
- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to
|
||||
- require a specific shape on every record. ZIP archives (.zip) are
|
||||
- validated structurally (data.jsonl present, media references resolve) in
|
||||
- addition to per-record content checks. Use --full-validate to JSON.parse
|
||||
- every line.
|
||||
- image = {img_path:"..."} (image generation);
|
||||
- video = {first_frame_path:"...", video_path:"..."} (video generation,
|
||||
- i2v first-frame or kf2v first+last-frame with last_frame_path). With no
|
||||
- --schema, a record carrying wav_fn is validated as TTS, img_path as image,
|
||||
- first_frame_path/video_path as video, chosen/rejected as DPO, text (no
|
||||
- messages) as CPT, otherwise ChatML. Pass --schema to require a specific
|
||||
- shape on every record. ZIP archives (.zip) are validated structurally
|
||||
- (data.jsonl present, media references resolve) in addition to per-record
|
||||
- content checks. Use --full-validate to JSON.parse every line.
|
||||
|
||||
#### Examples
|
||||
|
||||
@@ -217,6 +219,10 @@ bl dataset validate --file cpt.jsonl --schema cpt
|
||||
bl dataset validate --file audio.zip --schema tts
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset validate --file wan-i2v-training-dataset.zip --schema video
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset validate --file eval.jsonl --full-validate
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user