skills/modelstudioai__cli
1
0
Fork 0
mirror of https://github.com/modelstudioai/cli.git synced 2026-09-14 19:49:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: update datalist validate

Browse Source
This commit is contained in:
故璃
2026-08-11 13:33:31 +08:00
parent 5d1b7aac3a
commit 1c76749ee5
3 changed files with 53 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/commands/src/commands/dataset/upload.ts
+4 -11
View File
@@ -28,7 +28,7 @@ const UPLOAD_FLAGS = {
type: "string",
valueHint: "<s>",
description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
},
noValidate: {
type: "switch",
@@ -44,7 +44,7 @@ export default defineCommand({
description: "Upload a dataset file (.jsonl or .zip) to Bailian",
auth: "apiKey",
usageArgs:
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image>] [--no-validate] [--full-validate]",
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
flags: UPLOAD_FLAGS,
exampleArgs: [
"--file train.jsonl",
@@ -72,15 +72,8 @@ export default defineCommand({
const filePath = flags.file;
const purpose = flags.purpose || "fine-tune";
const schema = parseDatasetSchemaFlag(flags.schema);
if (schema === "video") {
throw new BailianError(
`--schema video is not supported.`,
ExitCode.USAGE,
`Supported schemas: chatml, dpo, cpt, tts, image.`,
);
}
// Image schema allows larger ZIPs (1 GB vs 300 MB for text).
const isMediaSchema = schema === "image";
// Image and video schemas allow larger ZIPs (1 GB vs 300 MB for text).
const isMediaSchema = schema === "image" || schema === "video";
if (!flags.noValidate) {
const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;
packages/commands/src/commands/dataset/validate.ts
+12 -16
View File
@@ -23,7 +23,7 @@ const VALIDATE_FLAGS = {
type: "string",
valueHint: "<s>",
description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
},
} satisfies FlagsDef;
@@ -31,13 +31,14 @@ export default defineCommand({
description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
// 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。
auth: "none",
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image>]",
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
flags: VALIDATE_FLAGS,
exampleArgs: [
"--file train.jsonl",
"--file dpo.jsonl --schema dpo",
"--file cpt.jsonl --schema cpt",
"--file audio.zip --schema tts",
"--file wan-i2v-training-dataset.zip --schema video",
"--file eval.jsonl --full-validate",
"--file train.jsonl --output json",
],
@@ -47,25 +48,20 @@ export default defineCommand({
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
'image = {img_path:"..."} (image generation). With no --schema, a record',
"carrying wav_fn is validated as TTS, img_path as image, chosen/rejected",
"as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to",
"require a specific shape on every record. ZIP archives (.zip) are",
"validated structurally (data.jsonl present, media references resolve) in",
"addition to per-record content checks. Use --full-validate to JSON.parse",
"every line.",
'image = {img_path:"..."} (image generation);',
'video = {first_frame_path:"...", video_path:"..."} (video generation,',
"i2v first-frame or kf2v first+last-frame with last_frame_path). With no",
"--schema, a record carrying wav_fn is validated as TTS, img_path as image,",
"first_frame_path/video_path as video, chosen/rejected as DPO, text (no",
"messages) as CPT, otherwise ChatML. Pass --schema to require a specific",
"shape on every record. ZIP archives (.zip) are validated structurally",
"(data.jsonl present, media references resolve) in addition to per-record",
"content checks. Use --full-validate to JSON.parse every line.",
],
async run(ctx) {
const { settings, flags } = ctx;
const filePath = flags.file;
const schema = parseDatasetSchemaFlag(flags.schema);
if (schema === "video") {
throw new BailianError(
`--schema video is not supported.`,
ExitCode.USAGE,
`Supported schemas: chatml, dpo, cpt, tts, image.`,
);
}
if (settings.dryRun) {
emitResult(
{
skills/bailian-finetune/reference/dataset.md
+37 -31
View File
@@ -107,23 +107,23 @@ bl dataset list --output json
### `bl dataset upload`
| Field | Value |
| --------------- | -------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `dataset upload` |
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image>] [--no-validate] [--full-validate]` |
| Field | Value |
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `dataset upload` |
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
#### Flags
| Flag | Type | Required | Description |
| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
| Flag | Type | Required | Description |
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
#### Notes
@@ -170,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate
### `bl dataset validate`
| Field | Value |
| --------------- | ----------------------------------------------------------------------------------------------- |
| **Name** | `dataset validate` |
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image>]` |
| Field | Value |
| --------------- | ------------------------------------------------------------------------------------------------------ |
| **Name** | `dataset validate` |
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
#### Flags
| Flag | Type | Required | Description |
| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
| Flag | Type | Required | Description |
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
#### Notes
@@ -191,13 +191,15 @@ bl dataset upload --file train.jsonl --no-validate
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
- rejected}; cpt = {text:"..."} (continual pre-training, raw text);
- tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
- image = {img_path:"..."} (image generation). With no --schema, a record
- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected
- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to
- require a specific shape on every record. ZIP archives (.zip) are
- validated structurally (data.jsonl present, media references resolve) in
- addition to per-record content checks. Use --full-validate to JSON.parse
- every line.
- image = {img_path:"..."} (image generation);
- video = {first_frame_path:"...", video_path:"..."} (video generation,
- i2v first-frame or kf2v first+last-frame with last_frame_path). With no
- --schema, a record carrying wav_fn is validated as TTS, img_path as image,
- first_frame_path/video_path as video, chosen/rejected as DPO, text (no
- messages) as CPT, otherwise ChatML. Pass --schema to require a specific
- shape on every record. ZIP archives (.zip) are validated structurally
- (data.jsonl present, media references resolve) in addition to per-record
- content checks. Use --full-validate to JSON.parse every line.
#### Examples
@@ -217,6 +219,10 @@ bl dataset validate --file cpt.jsonl --schema cpt
bl dataset validate --file audio.zip --schema tts
```
```bash
bl dataset validate --file wan-i2v-training-dataset.zip --schema video
```
```bash
bl dataset validate --file eval.jsonl --full-validate
```