diff --git a/packages/commands/src/commands/dataset/upload.ts b/packages/commands/src/commands/dataset/upload.ts index 1363509..6888b11 100644 --- a/packages/commands/src/commands/dataset/upload.ts +++ b/packages/commands/src/commands/dataset/upload.ts @@ -28,7 +28,7 @@ const UPLOAD_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', }, noValidate: { type: "switch", @@ -44,7 +44,7 @@ export default defineCommand({ description: "Upload a dataset file (.jsonl or .zip) to Bailian", auth: "apiKey", usageArgs: - "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", + "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", flags: UPLOAD_FLAGS, exampleArgs: [ "--file train.jsonl", @@ -72,15 +72,8 @@ export default defineCommand({ const filePath = flags.file; const purpose = flags.purpose || "fine-tune"; const schema = parseDatasetSchemaFlag(flags.schema); - if (schema === "video") { - throw new BailianError( - `--schema video is not supported.`, - ExitCode.USAGE, - `Supported schemas: chatml, dpo, cpt, tts, image.`, - ); - } - // Image schema allows larger ZIPs (1 GB vs 300 MB for text). - const isMediaSchema = schema === "image"; + // Image and video schemas allow larger ZIPs (1 GB vs 300 MB for text). + const isMediaSchema = schema === "image" || schema === "video"; if (!flags.noValidate) { const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES; diff --git a/packages/commands/src/commands/dataset/validate.ts b/packages/commands/src/commands/dataset/validate.ts index 142fbdf..b9e7f62 100644 --- a/packages/commands/src/commands/dataset/validate.ts +++ b/packages/commands/src/commands/dataset/validate.ts @@ -23,7 +23,7 @@ const VALIDATE_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', }, } satisfies FlagsDef; @@ -31,13 +31,14 @@ export default defineCommand({ description: "Locally validate a dataset file (.jsonl or .zip) without uploading", // 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。 auth: "none", - usageArgs: "--file [--full-validate] [--schema ]", + usageArgs: "--file [--full-validate] [--schema ]", flags: VALIDATE_FLAGS, exampleArgs: [ "--file train.jsonl", "--file dpo.jsonl --schema dpo", "--file cpt.jsonl --schema cpt", "--file audio.zip --schema tts", + "--file wan-i2v-training-dataset.zip --schema video", "--file eval.jsonl --full-validate", "--file train.jsonl --output json", ], @@ -47,25 +48,20 @@ export default defineCommand({ "Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,", 'rejected}; cpt = {text:"..."} (continual pre-training, raw text);', 'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);', - 'image = {img_path:"..."} (image generation). With no --schema, a record', - "carrying wav_fn is validated as TTS, img_path as image, chosen/rejected", - "as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to", - "require a specific shape on every record. ZIP archives (.zip) are", - "validated structurally (data.jsonl present, media references resolve) in", - "addition to per-record content checks. Use --full-validate to JSON.parse", - "every line.", + 'image = {img_path:"..."} (image generation);', + 'video = {first_frame_path:"...", video_path:"..."} (video generation,', + "i2v first-frame or kf2v first+last-frame with last_frame_path). With no", + "--schema, a record carrying wav_fn is validated as TTS, img_path as image,", + "first_frame_path/video_path as video, chosen/rejected as DPO, text (no", + "messages) as CPT, otherwise ChatML. Pass --schema to require a specific", + "shape on every record. ZIP archives (.zip) are validated structurally", + "(data.jsonl present, media references resolve) in addition to per-record", + "content checks. Use --full-validate to JSON.parse every line.", ], async run(ctx) { const { settings, flags } = ctx; const filePath = flags.file; const schema = parseDatasetSchemaFlag(flags.schema); - if (schema === "video") { - throw new BailianError( - `--schema video is not supported.`, - ExitCode.USAGE, - `Supported schemas: chatml, dpo, cpt, tts, image.`, - ); - } if (settings.dryRun) { emitResult( { diff --git a/skills/bailian-finetune/reference/dataset.md b/skills/bailian-finetune/reference/dataset.md index 523a421..22cad6d 100644 --- a/skills/bailian-finetune/reference/dataset.md +++ b/skills/bailian-finetune/reference/dataset.md @@ -107,23 +107,23 @@ bl dataset list --output json ### `bl dataset upload` -| Field | Value | -| --------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `dataset upload` | -| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian | -| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | +| Field | Value | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `dataset upload` | +| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian | +| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | #### Flags -| Flag | Type | Required | Description | -| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) | -| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | -| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) | +| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | +| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes @@ -170,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate ### `bl dataset validate` -| Field | Value | -| --------------- | ----------------------------------------------------------------------------------------------- | -| **Name** | `dataset validate` | -| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | -| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | +| Field | Value | +| --------------- | ------------------------------------------------------------------------------------------------------ | +| **Name** | `dataset validate` | +| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | +| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | #### Flags -| Flag | Type | Required | Description | -| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | +| Flag | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | #### Notes @@ -191,13 +191,15 @@ bl dataset upload --file train.jsonl --no-validate - Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen, - rejected}; cpt = {text:"..."} (continual pre-training, raw text); - tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); -- image = {img_path:"..."} (image generation). With no --schema, a record -- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected -- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to -- require a specific shape on every record. ZIP archives (.zip) are -- validated structurally (data.jsonl present, media references resolve) in -- addition to per-record content checks. Use --full-validate to JSON.parse -- every line. +- image = {img_path:"..."} (image generation); +- video = {first_frame_path:"...", video_path:"..."} (video generation, +- i2v first-frame or kf2v first+last-frame with last_frame_path). With no +- --schema, a record carrying wav_fn is validated as TTS, img_path as image, +- first_frame_path/video_path as video, chosen/rejected as DPO, text (no +- messages) as CPT, otherwise ChatML. Pass --schema to require a specific +- shape on every record. ZIP archives (.zip) are validated structurally +- (data.jsonl present, media references resolve) in addition to per-record +- content checks. Use --full-validate to JSON.parse every line. #### Examples @@ -217,6 +219,10 @@ bl dataset validate --file cpt.jsonl --schema cpt bl dataset validate --file audio.zip --schema tts ``` +```bash +bl dataset validate --file wan-i2v-training-dataset.zip --schema video +``` + ```bash bl dataset validate --file eval.jsonl --full-validate ```