feat: merge self-built-framework

This commit is contained in:
故璃
2026-07-09 10:02:56 +08:00
parent 49095c3a8a
commit 8fd072bcd1
42 changed files with 2120 additions and 232 deletions
+11
View File
@@ -104,6 +104,17 @@ CLI 只为「自己能权威解释的错误」发出语义化信号,服务端的
如果命令调用 Console Gateway,`defineCommand` 必须设置 `auth: "console"`。runtime 会基于 `CONSOLE_AUTH_FLAGS` 自动在 help 中展示 `--console-region``--console-site``--console-switch-agent``--workspace-id`,并由 `authStage` 解析/注入 console credential。命令不要重复声明这些凭证域 flag,也不要手动从 env/config 解析 token。 如果命令调用 Console Gateway,`defineCommand` 必须设置 `auth: "console"`。runtime 会基于 `CONSOLE_AUTH_FLAGS` 自动在 help 中展示 `--console-region``--console-site``--console-switch-agent``--workspace-id`,并由 `authStage` 解析/注入 console credential。命令不要重复声明这些凭证域 flag,也不要手动从 env/config 解析 token。
### 5. 禁止单字母变量命名
所有变量、参数、回调形参必须使用有语义的命名,不允许单字母(如 `i``m``p``t``e``s`)。具体表现:
- 回调参数: `.map((m) => ...)``.map((model) => ...)`, `.find((t) => ...)``.find((template) => ...)`
- catch 变量: `catch (e)``catch (error)`
- for-of 循环: `for (const i of items)``for (const item of items)`
- 临时变量: `const s = ...``const strategy = ...`
例外: 仅当作用域极小(≤3 行)且语义从上下文完全明确时,可使用 `k`/`v`(Object.entries 的 key/value)。
## 完成改动后的快速验证 ## 完成改动后的快速验证
```sh ```sh
@@ -214,6 +214,52 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => {
expect(data.action).toBe("dataset.upload"); expect(data.action).toBe("dataset.upload");
expect(data.schema).toBe("dpo"); expect(data.schema).toBe("dpo");
}); });
test("dataset upload --schema image --no-validate --dry-run 采用 1GB 媒体上限", async () => {
// image/video schemas raise the upload cap to 1 GiB (vs 300 MB for text).
// --no-validate keeps this offline (the jsonl fixture is not a real zip).
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"upload",
"--file",
file,
"--schema",
"image",
"--no-validate",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{ action: string; schema: string; max_bytes: number }>(stdout);
expect(data.action).toBe("dataset.upload");
expect(data.schema).toBe("image");
expect(data.max_bytes).toBe(1024 * 1024 * 1024);
});
test.each(["tts", "image", "video"])(
"dataset upload --dry-run 接受媒体 schema %s",
async (schema) => {
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"upload",
"--file",
file,
"--schema",
schema,
"--no-validate",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{ action: string; schema: string }>(stdout);
expect(data.action).toBe("dataset.upload");
expect(data.schema).toBe(schema);
},
);
}); });
describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => { describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => {
+92
View File
@@ -56,6 +56,98 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => {
expect(data.body.capacity).toBe(1); expect(data.body.capacity).toBe(1);
}); });
test("deploy create --dry-run 组装视频 LoRA 的 aigc_config", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
"create",
"--model",
"wan2.5-i2v-preview-ft-xxx",
"--name",
"my-video-lora",
"--aigc-prompt",
"a cat surfing",
"--aigc-lora-prompt-default",
"trigger-word",
"--aigc-use-input-prompt",
"true",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{
action: string;
body: {
plan: string;
aigc_config?: {
use_input_prompt?: boolean;
prompt?: string;
lora_prompt_default?: string;
};
};
}>(stdout);
expect(data.action).toBe("deploy.create");
expect(data.body.plan).toBe("lora");
expect(data.body.aigc_config).toEqual({
use_input_prompt: true,
prompt: "a cat surfing",
lora_prompt_default: "trigger-word",
});
});
test("deploy create --aigc-* 仅对 plan=lora 有效plan=ptu 时报错)", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
"create",
"--model",
"wan2.5-i2v-preview-ft-xxx",
"--name",
"my-video-lora",
"--plan",
"ptu",
"--input-tpm",
"10000",
"--output-tpm",
"1000",
"--aigc-prompt",
"a cat surfing",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stdout + stderr).not.toBe(0);
expect(`${stdout}\n${stderr}`).toMatch(/--aigc-\* flags are only valid for plan=lora/);
});
test("deploy create --plan mu --deploy-spec --dry-run 透传 deploy_spec", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
"create",
"--model",
"qwen3-8b",
"--name",
"my-qwen3-mu",
"--plan",
"mu",
"--deploy-spec",
"MU1",
"--capacity",
"2",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{
action: string;
body: { plan: string; deploy_spec?: string; capacity?: number };
}>(stdout);
expect(data.action).toBe("deploy.create");
expect(data.body.plan).toBe("mu");
expect(data.body.deploy_spec).toBe("MU1");
expect(data.body.capacity).toBe(2);
});
test("deploy scale --dry-run 转发 capacity", async () => { test("deploy scale --dry-run 转发 capacity", async () => {
const { stdout, stderr, exitCode } = await runCli([ const { stdout, stderr, exitCode } = await runCli([
"deploy", "deploy",
@@ -114,6 +114,35 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => {
}); });
}); });
test.each([
["sft", "sft"],
["sft-lora", "efficient_sft"],
["dpo", "dpo_full"],
["dpo-lora", "dpo_lora"],
["cpt", "cpt"],
])(
"finetune create --training-type %s 经 profile 映射为 server 类型 %s",
async (cliType, serverType) => {
const { stdout, stderr, exitCode } = await runCli([
"finetune",
"create",
"--model",
"qwen3-8b",
"--datasets",
"file-aaa",
"--training-type",
cliType,
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{ action: string; body: { training_type: string } }>(stdout);
expect(data.action).toBe("finetune.create");
expect(data.body.training_type).toBe(serverType);
},
);
test("finetune create --training-type 拒绝不支持的训练类型值", async () => { test("finetune create --training-type 拒绝不支持的训练类型值", async () => {
const { stdout, stderr, exitCode } = await runCli([ const { stdout, stderr, exitCode } = await runCli([
"finetune", "finetune",
@@ -6,6 +6,7 @@ import {
parseDatasetSchemaFlag, parseDatasetSchemaFlag,
formatIssue, formatIssue,
MAX_DATASET_BYTES, MAX_DATASET_BYTES,
MAX_MEDIA_ZIP_BYTES,
BailianError, BailianError,
ExitCode, ExitCode,
type DatasetFile, type DatasetFile,
@@ -17,7 +18,7 @@ const UPLOAD_FLAGS = {
file: { file: {
type: "string", type: "string",
valueHint: "<path>", valueHint: "<path>",
description: "Local .jsonl dataset file (≤300MB)", description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video)",
required: true, required: true,
}, },
purpose: { purpose: {
@@ -29,7 +30,7 @@ const UPLOAD_FLAGS = {
type: "string", type: "string",
valueHint: "<s>", valueHint: "<s>",
description: description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record.', 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
}, },
noValidate: { noValidate: {
type: "switch", type: "switch",
@@ -42,30 +43,31 @@ const UPLOAD_FLAGS = {
} satisfies FlagsDef; } satisfies FlagsDef;
export default defineCommand({ export default defineCommand({
description: "Upload a dataset file (.jsonl) to Bailian", description: "Upload a dataset file (.jsonl or .zip) to Bailian",
auth: "apiKey", auth: "apiKey",
usageArgs: usageArgs:
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt>] [--no-validate] [--full-validate]", "--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
flags: UPLOAD_FLAGS, flags: UPLOAD_FLAGS,
exampleArgs: [ exampleArgs: [
"--file train.jsonl", "--file train.jsonl",
"--file dpo.jsonl --schema dpo", "--file dpo.jsonl --schema dpo",
"--file cpt.jsonl --schema cpt", "--file cpt.jsonl --schema cpt",
"--file audio.zip --schema tts",
"--file eval.jsonl --purpose evaluation", "--file eval.jsonl --purpose evaluation",
"--file train.jsonl --full-validate", "--file train.jsonl --full-validate",
"--file train.jsonl --no-validate", "--file train.jsonl --no-validate",
], ],
notes: [ notes: [
"Only .jsonl is supported in this release. Three record schemas are", "Supports .jsonl (text) and .zip (audio/image/video archives with a",
"recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],", "data.jsonl manifest). Six record schemas are recognized: chatml =",
"chosen, rejected} where chosen/rejected are single assistant messages;", "{messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};",
'cpt = {text:"..."} (continual pre-training, raw text). With no --schema,', 'cpt = {text:"..."} (continual pre-training, raw text); tts =',
"a record carrying chosen/rejected is validated as DPO, one with text (and", '{wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =',
"no messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to", '{img_path:"..."} (image generation); video = {first_frame_path:"...",',
"require that shape on every record, or --schema chatml to ignore the", 'video_path:"..."} (video generation). With no --schema, a record',
"preference / text fields. Other purposes may carry a different schema in", "carrying wav_fn is validated as TTS, img_path as image, video_path /",
"the future and would be served by a purpose-specific validator.", "first_frame_path as video, chosen/rejected as DPO, text (no messages)",
"The dataset upload cap is 300MB per file.", "as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.",
"Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so", "Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so",
"the purpose tag is persisted (the DashScope-native /api/v1/files drops it).", "the purpose tag is persisted (the DashScope-native /api/v1/files drops it).",
], ],
@@ -75,9 +77,16 @@ export default defineCommand({
const purpose = flags.purpose || "fine-tune"; const purpose = flags.purpose || "fine-tune";
const schema = parseDatasetSchemaFlag(flags.schema); const schema = parseDatasetSchemaFlag(flags.schema);
const format = detectOutputFormat(settings.output); const format = detectOutputFormat(settings.output);
// Image / video schemas allow larger ZIPs (1 GB vs 300 MB for text).
const isMediaSchema = schema === "image" || schema === "video";
if (!flags.noValidate) { if (!flags.noValidate) {
const result = await validateDataset(filePath, { fullValidate: flags.fullValidate, schema }); const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;
const result = await validateDataset(filePath, {
fullValidate: flags.fullValidate,
schema,
maxBytes,
});
if (!result.valid) { if (!result.valid) {
const lines = [ const lines = [
`Dataset validation failed for ${filePath}`, `Dataset validation failed for ${filePath}`,
@@ -112,7 +121,7 @@ export default defineCommand({
action: "dataset.upload", action: "dataset.upload",
file: filePath, file: filePath,
purpose, purpose,
max_bytes: MAX_DATASET_BYTES, max_bytes: isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES,
validate: !flags.noValidate, validate: !flags.noValidate,
schema: schema ?? "auto", schema: schema ?? "auto",
}, },
@@ -25,7 +25,7 @@ const VALIDATE_FLAGS = {
file: { file: {
type: "string", type: "string",
valueHint: "<path>", valueHint: "<path>",
description: "Local .jsonl dataset file", description: "Local dataset file (.jsonl or .zip)",
required: true, required: true,
}, },
fullValidate: { fullValidate: {
@@ -36,20 +36,21 @@ const VALIDATE_FLAGS = {
type: "string", type: "string",
valueHint: "<s>", valueHint: "<s>",
description: description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record.', 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
}, },
} satisfies FlagsDef; } satisfies FlagsDef;
export default defineCommand({ export default defineCommand({
description: "Locally validate a dataset file (.jsonl) without uploading", description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
// 纯本地校验,不触网、不需 API key与 `pipeline validate` 一致)。 // 纯本地校验,不触网、不需 API key与 `pipeline validate` 一致)。
auth: "none", auth: "none",
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt>]", usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
flags: VALIDATE_FLAGS, flags: VALIDATE_FLAGS,
exampleArgs: [ exampleArgs: [
"--file train.jsonl", "--file train.jsonl",
"--file dpo.jsonl --schema dpo", "--file dpo.jsonl --schema dpo",
"--file cpt.jsonl --schema cpt", "--file cpt.jsonl --schema cpt",
"--file audio.zip --schema tts",
"--file eval.jsonl --full-validate", "--file eval.jsonl --full-validate",
"--file train.jsonl --output json", "--file train.jsonl --output json",
], ],
@@ -57,12 +58,16 @@ export default defineCommand({
"Default scan: every line gets a structural check, then ~160 lines (front 50,", "Default scan: every line gets a structural check, then ~160 lines (front 50,",
"evenly spaced 100, last 10) are JSON.parsed against the active schema.", "evenly spaced 100, last 10) are JSON.parsed against the active schema.",
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,", "Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
"rejected} where chosen/rejected are single assistant messages; cpt =", 'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
'{text:"..."} (continual pre-training, raw text). With no --schema, a', 'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
"record carrying chosen/rejected is validated as DPO, one with text (and no", 'image = {img_path:"..."} (image generation); video =',
"messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to require", '{first_frame_path:"...", video_path:"..."} (video generation). With no',
"that shape on every record (strict), or --schema chatml to ignore the", "--schema, a record carrying wav_fn is validated as TTS, img_path as",
"preference / text fields. Use --full-validate to JSON.parse every line.", "image, video_path / first_frame_path as video, chosen/rejected as DPO,",
"text (no messages) as CPT, otherwise ChatML. Pass --schema to require a",
"specific shape on every record. ZIP archives (.zip) are validated",
"structurally (data.jsonl present, media references resolve) in addition",
"to per-record content checks. Use --full-validate to JSON.parse every line.",
], ],
async run(ctx) { async run(ctx) {
const { settings, flags } = ctx; const { settings, flags } = ctx;
@@ -2,6 +2,8 @@ import {
defineCommand, defineCommand,
detectOutputFormat, detectOutputFormat,
createDeployment, createDeployment,
BailianError,
ExitCode,
type CreateDeploymentRequest, type CreateDeploymentRequest,
type FlagsDef, type FlagsDef,
} from "bailian-cli-core"; } from "bailian-cli-core";
@@ -26,10 +28,10 @@ const CREATE_FLAGS = {
valueHint: "<plan>", valueHint: "<plan>",
description: "Billing plan: lora (default, Token-billed) | ptu (Token-billed) | mu", description: "Billing plan: lora (default, Token-billed) | ptu (Token-billed) | mu",
}, },
templateId: { deploySpec: {
type: "string", type: "string",
valueHint: "<id>", valueHint: "<id>",
description: "Template id (only used by plan=mu; auto-picked if omitted)", description: "Deploy spec (only used by plan=mu; auto-picked if omitted)",
}, },
capacity: { capacity: {
type: "number", type: "number",
@@ -56,6 +58,23 @@ const CREATE_FLAGS = {
valueHint: "<n>", valueHint: "<n>",
description: "PTU max thinking-output tokens/min (optional, some models)", description: "PTU max thinking-output tokens/min (optional, some models)",
}, },
aigcUseInputPrompt: {
type: "boolean",
valueHint: "<bool>",
description:
"Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template)",
},
aigcPrompt: {
type: "string",
valueHint: "<text>",
description:
"Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false",
},
aigcLoraPromptDefault: {
type: "string",
valueHint: "<text>",
description: "Video LoRA (aigc_config): default trigger-word phrase for the LoRA",
},
} satisfies FlagsDef; } satisfies FlagsDef;
/** /**
@@ -73,25 +92,29 @@ export default defineCommand({
description: "Create a model deployment", description: "Create a model deployment",
auth: "apiKey", auth: "apiKey",
usageArgs: usageArgs:
"--model <model_name> --name <display_name> [--plan <plan>] [--template-id <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]", "--model <model_name> --name <display_name> [--plan <plan>] [--deploy-spec <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]",
flags: CREATE_FLAGS, flags: CREATE_FLAGS,
exampleArgs: [ exampleArgs: [
"--model my-qwen-sft --name my-sft-test", "--model my-qwen-sft --name my-sft-test",
"--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000", "--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000",
"--model qwen3-8b --name my-qwen3-mu --plan mu", "--model qwen3-8b --name my-qwen3-mu --plan mu",
"--model qwen3-8b --name my-qwen3 --plan mu --template-id MU1 --capacity 2", "--model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2",
'--model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."',
], ],
notes: [ notes: [
"Plan defaults to `lora` (Token-billed). Pass --plan to override.", "Plan defaults to `lora` (Token-billed). Pass --plan to override.",
"For plan=ptu (Token-billed, provisioned throughput), --input-tpm and", "For plan=ptu (Token-billed, provisioned throughput), --input-tpm and",
"--output-tpm are required (the platform rejects creation without an", "--output-tpm are required (the platform rejects creation without an",
"explicit ptu_capacity despite the doc listing defaults).", "explicit ptu_capacity despite the doc listing defaults).",
"For plan=mu, `capacity`, `billing_method` and `template_id` are required.", "For plan=mu, `capacity`, `billing_method` and `deploy_spec` are required.",
"billing_method defaults to POST_PAY (only supported value); template_id", "billing_method defaults to POST_PAY (only supported value); deploy_spec",
"and capacity are auto-picked from GET /deployments/models when omitted.", "and capacity are auto-picked from GET /deployments/models when omitted.",
"Use `bl deploy models --source base` to inspect available templates.", "Use `bl deploy models --source base` to inspect available templates.",
"After creation, status starts at PENDING and transitions to RUNNING.", "After creation, status starts at PENDING and transitions to RUNNING.",
"Invoke the deployed model with: bl text chat --model <deployed_model>", "Invoke the deployed model with: bl text chat --model <deployed_model>",
"For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass",
"--aigc-prompt / --aigc-lora-prompt-default (and optionally",
"--aigc-use-input-prompt) to set the deployment's aigc_config.",
"WARNING: --model is overloaded across commands and refers to DIFFERENT", "WARNING: --model is overloaded across commands and refers to DIFFERENT",
"values. `bl deploy create --model` takes the exported model_name (e.g.", "values. `bl deploy create --model` takes the exported model_name (e.g.",
"`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`", "`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`",
@@ -136,6 +159,33 @@ export default defineCommand({
...resolved.body, ...resolved.body,
}; };
// AIGC config (fine-tuned Wan video LoRA deployments). Only valid for
// plan=lora — reject early for ptu/mu so the user gets a clear CLI error
// instead of an opaque server-side rejection.
const aigcUseInputPrompt = flags.aigcUseInputPrompt;
const aigcPrompt = flags.aigcPrompt;
const aigcLoraPromptDefault = flags.aigcLoraPromptDefault;
const hasAigcFlags =
aigcUseInputPrompt !== undefined ||
aigcPrompt !== undefined ||
aigcLoraPromptDefault !== undefined;
if (hasAigcFlags && plan !== "lora") {
throw new BailianError(
`--aigc-* flags are only valid for plan=lora (video LoRA deployments). Got plan=${plan}.`,
ExitCode.USAGE,
);
}
if (hasAigcFlags) {
const aigcConfig: Record<string, unknown> = {
use_input_prompt: aigcUseInputPrompt ?? false,
};
if (aigcPrompt !== undefined) aigcConfig.prompt = aigcPrompt;
if (aigcLoraPromptDefault !== undefined) {
aigcConfig.lora_prompt_default = aigcLoraPromptDefault;
}
body.aigc_config = aigcConfig;
}
if (settings.dryRun) { if (settings.dryRun) {
emitResult({ action: "deploy.create", body }, format); emitResult({ action: "deploy.create", body }, format);
return; return;
@@ -59,8 +59,8 @@ export default defineCommand({
ExitCode.USAGE, ExitCode.USAGE,
); );
} }
} catch (e) { } catch (error) {
if (e instanceof BailianError) throw e; if (error instanceof BailianError) throw error;
// If the get itself failed (e.g. not found), let the DELETE call surface the real error. // If the get itself failed (e.g. not found), let the DELETE call surface the real error.
} }
} }
@@ -68,13 +68,13 @@ export default defineCommand({
return; return;
} }
const headers = ["DEPLOYED_MODEL", "MODEL_NAME", "STATUS", "PLAN", "CAPACITY", "CREATED_AT"]; const headers = ["DEPLOYED_MODEL", "MODEL_NAME", "STATUS", "PLAN", "CAPACITY", "CREATED_AT"];
const rows = items.map((i) => [ const rows = items.map((item) => [
i.deployed_model, item.deployed_model,
i.model_name, item.model_name,
i.status, item.status,
i.plan, item.plan,
i.capacity, item.capacity,
i.created_at, item.created_at,
]); ]);
for (const line of formatTable(headers, rows)) emitBare(line); for (const line of formatTable(headers, rows)) emitBare(line);
if (total !== undefined) emitBare(`\nTotal: ${total}`); if (total !== undefined) emitBare(`\nTotal: ${total}`);
+41 -36
View File
@@ -76,44 +76,44 @@ export default defineCommand({
// downstream tooling can drive `bl deploy create --template-id <…>` without // downstream tooling can drive `bl deploy create --template-id <…>` without
// a second round-trip. For text: keep the compact one-line summary. // a second round-trip. For text: keep the compact one-line summary.
if (format === "json") { if (format === "json") {
const items = models.map((m) => { const items = models.map((model) => {
const out: Record<string, unknown> = { const out: Record<string, unknown> = {
model_name: m.model_name ?? "", model_name: model.model_name ?? "",
}; };
if (m.base_model) out.base_model = m.base_model; if (model.base_model) out.base_model = model.base_model;
if (m.model_source) out.model_source = m.model_source; if (model.model_source) out.model_source = model.model_source;
if (m.supported_plans && m.supported_plans.length > 0) { if (model.supported_plans && model.supported_plans.length > 0) {
out.supported_plans = m.supported_plans; out.supported_plans = model.supported_plans;
} }
if (m.plans && m.plans.length > 0) { if (model.plans && model.plans.length > 0) {
out.plans = m.plans.map((p) => { out.plans = model.plans.map((plan) => {
const planEntry: Record<string, unknown> = { plan: p.plan ?? "" }; const planEntry: Record<string, unknown> = { plan: plan.plan ?? "" };
if (p.cu_specs && p.cu_specs.length > 0) { if (plan.cu_specs && plan.cu_specs.length > 0) {
planEntry.cu_specs = p.cu_specs; planEntry.cu_specs = plan.cu_specs;
} }
if (p.templates && p.templates.length > 0) { if (plan.templates && plan.templates.length > 0) {
// Pull the top 6 fields most useful for `bl deploy create`. // Pull the top 6 fields most useful for `bl deploy create`.
// Drop noisy/redundant: template_source, template_type, // Drop noisy/redundant: template_source, template_type,
// template_version, deploy_spec (typically == template_id). // template_version, deploy_spec (typically == template_id).
planEntry.templates = p.templates.map((t) => { planEntry.templates = plan.templates.map((template) => {
const tpl: Record<string, unknown> = {}; const tpl: Record<string, unknown> = {};
if (t.template_id) tpl.template_id = t.template_id; if (template.template_id) tpl.template_id = template.template_id;
if (t.template_name) tpl.template_name = t.template_name; if (template.template_name) tpl.template_name = template.template_name;
if (t.charge_type) tpl.charge_type = t.charge_type; if (template.charge_type) tpl.charge_type = template.charge_type;
// Flatten roles.unified for the common COUPLED case. // Flatten roles.unified for the common COUPLED case.
const unified = t.roles?.unified; const unified = template.roles?.unified;
if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec; if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec;
if (unified?.capacity_unit_per_instance !== undefined) if (unified?.capacity_unit_per_instance !== undefined)
tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance; tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance;
// Preserve split-role configs (SEPERATED) as-is so callers // Preserve split-role configs (SEPERATED) as-is so callers
// can still drive prefill/decode sizing. // can still drive prefill/decode sizing.
if (t.roles?.prefill || t.roles?.decode) { if (template.roles?.prefill || template.roles?.decode) {
tpl.roles = { tpl.roles = {
prefill: t.roles?.prefill, prefill: template.roles?.prefill,
decode: t.roles?.decode, decode: template.roles?.decode,
}; };
} }
if (t.template_desc) tpl.template_desc = t.template_desc; if (template.template_desc) tpl.template_desc = template.template_desc;
return tpl; return tpl;
}); });
} }
@@ -127,19 +127,19 @@ export default defineCommand({
} }
// text / quiet — keep the compact single-line summary table. // text / quiet — keep the compact single-line summary table.
const textItems = models.map((m) => { const textItems = models.map((model) => {
let plansSummary = ""; let plansSummary = "";
if (m.supported_plans && m.supported_plans.length > 0) { if (model.supported_plans && model.supported_plans.length > 0) {
plansSummary = m.supported_plans.join(","); plansSummary = model.supported_plans.join(",");
} else if (m.plans && m.plans.length > 0) { } else if (model.plans && model.plans.length > 0) {
plansSummary = m.plans plansSummary = model.plans
.map((p) => { .map((plan) => {
const planName = p.plan ?? "?"; const planName = plan.plan ?? "?";
if (p.templates && p.templates.length > 0) { if (plan.templates && plan.templates.length > 0) {
return `${planName}(${p.templates.length}t)`; return `${planName}(${plan.templates.length}t)`;
} }
if (p.cu_specs && p.cu_specs.length > 0) { if (plan.cu_specs && plan.cu_specs.length > 0) {
return `${planName}(${p.cu_specs.join("/")})`; return `${planName}(${plan.cu_specs.join("/")})`;
} }
return planName; return planName;
}) })
@@ -148,9 +148,9 @@ export default defineCommand({
plansSummary = "-"; plansSummary = "-";
} }
return { return {
model_name: m.model_name ?? "", model_name: model.model_name ?? "",
base_model: m.base_model ?? "", base_model: model.base_model ?? "",
source: m.model_source ?? "", source: model.model_source ?? "",
plans: plansSummary, plans: plansSummary,
}; };
}); });
@@ -160,7 +160,12 @@ export default defineCommand({
return; return;
} }
const headers = ["MODEL_NAME", "BASE_MODEL", "SOURCE", "PLANS"]; const headers = ["MODEL_NAME", "BASE_MODEL", "SOURCE", "PLANS"];
const rows = textItems.map((i) => [i.model_name, i.base_model, i.source, i.plans]); const rows = textItems.map((item) => [
item.model_name,
item.base_model,
item.source,
item.plans,
]);
for (const line of formatTable(headers, rows)) emitBare(line); for (const line of formatTable(headers, rows)) emitBare(line);
if (total !== undefined) emitBare(`\nTotal: ${total}`); if (total !== undefined) emitBare(`\nTotal: ${total}`);
}, },
+21 -20
View File
@@ -18,7 +18,7 @@ import { listDeployableModels, BailianError, ExitCode, type Client } from "baili
/** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */ /** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */
export interface CreatePlanFlags { export interface CreatePlanFlags {
plan?: string; plan?: string;
templateId?: string; deploySpec?: string;
capacity?: number; capacity?: number;
billingMethod?: string; billingMethod?: string;
inputTpm?: number; inputTpm?: number;
@@ -101,15 +101,15 @@ const ptuStrategy: PlanStrategy = {
}; };
/** /**
* `mu` (model-unit-billed). `capacity`, `billing_method` and `template_id` are * `mu` (model-unit-billed). `capacity`, `billing_method` and `deploy_spec` are
* all required by the API but every one has a CLI-side default: * all required by the API but every one has a CLI-side default:
* - billing_method defaults to POST_PAY (the only supported value). * - billing_method defaults to POST_PAY (the only supported value).
* - template_id auto-picks from GET /deployments/models — the one whose * - deploy_spec auto-picks from GET /deployments/models — the one whose
* `charge_type` matches `billing_method`, else the first available. * `charge_type` matches `billing_method`, else the first available.
* - capacity defaults to the template's `capacity_unit_per_instance` (the * - capacity defaults to the template's `capacity_unit_per_instance` (the
* smallest valid multiple of base_capacity). * smallest valid multiple of base_capacity).
* *
* The catalog lookup is skipped when `--template-id` is supplied explicitly: * The catalog lookup is skipped when `--deploy-spec` is supplied explicitly:
* fine-tuned custom models may not appear in the `source=base` catalog, and * fine-tuned custom models may not appear in the `source=base` catalog, and
* forcing the lookup would otherwise raise a spurious "no template" error. * forcing the lookup would otherwise raise a spurious "no template" error.
* It is also skipped in dry-run mode to keep `--dry-run` side-effect-free. * It is also skipped in dry-run mode to keep `--dry-run` side-effect-free.
@@ -121,15 +121,15 @@ const muStrategy: PlanStrategy = {
}, },
async resolve(ctx: PlanContext): Promise<PlanResolved> { async resolve(ctx: PlanContext): Promise<PlanResolved> {
const billingMethod = ctx.flags.billingMethod || "POST_PAY"; const billingMethod = ctx.flags.billingMethod || "POST_PAY";
let templateId = ctx.flags.templateId; let deploySpec = ctx.flags.deploySpec;
let capacity = ctx.flags.capacity; let capacity = ctx.flags.capacity;
if (!ctx.dryRun && !templateId) { if (!ctx.dryRun && !deploySpec) {
const noTemplateError = () => const noTemplateError = () =>
new BailianError( new BailianError(
`No mu-plan template found for model "${ctx.model}". ` + `No mu-plan template found for model "${ctx.model}". ` +
`Run \`${ctx.binName} deploy models --source base\` to inspect available models, ` + `Run \`${ctx.binName} deploy models --source base\` to inspect available models, ` +
`or pass --template-id explicitly.`, `or pass --deploy-spec explicitly.`,
ExitCode.USAGE, ExitCode.USAGE,
); );
try { try {
@@ -139,23 +139,24 @@ const muStrategy: PlanStrategy = {
version: "v1.0", version: "v1.0",
}); });
const payload = resp.output ?? resp.data; const payload = resp.output ?? resp.data;
const target = (payload?.models ?? []).find((m) => m.model_name === ctx.model); const target = (payload?.models ?? []).find((model) => model.model_name === ctx.model);
const muPlan = target?.plans?.find((p) => p.plan === "mu"); const muPlan = target?.plans?.find((plan) => plan.plan === "mu");
const templates = muPlan?.templates ?? []; const templates = muPlan?.templates ?? [];
if (templates.length === 0) throw noTemplateError(); if (templates.length === 0) throw noTemplateError();
// POST_PAY → post_paid template; fall back to the first available. // POST_PAY → post_paid template; fall back to the first available.
const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid"; const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid";
const picked = templates.find((t) => t.charge_type === wantChargeType) ?? templates[0]; const picked =
if (!picked?.template_id) throw noTemplateError(); templates.find((template) => template.charge_type === wantChargeType) ?? templates[0];
templateId = picked.template_id; if (!picked?.deploy_spec && !picked?.template_id) throw noTemplateError();
deploySpec = picked.deploy_spec ?? picked.template_id;
if (capacity === undefined) { if (capacity === undefined) {
capacity = picked.roles?.unified?.capacity_unit_per_instance ?? 1; capacity = picked.roles?.unified?.capacity_unit_per_instance ?? 1;
} }
} catch (e) { } catch (error) {
if (e instanceof BailianError) throw e; if (error instanceof BailianError) throw error;
throw new BailianError( throw new BailianError(
`Failed to auto-pick template for plan=mu: ${(e as Error).message}. ` + `Failed to auto-pick template for plan=mu: ${(error as Error).message}. ` +
`Pass --template-id explicitly.`, `Pass --deploy-spec explicitly.`,
ExitCode.USAGE, ExitCode.USAGE,
); );
} }
@@ -165,7 +166,7 @@ const muStrategy: PlanStrategy = {
capacity: capacity ?? 1, capacity: capacity ?? 1,
billing_method: billingMethod, billing_method: billingMethod,
}; };
if (templateId) body.template_id = templateId; if (deploySpec) body.deploy_spec = deploySpec;
return { body }; return { body };
}, },
}; };
@@ -184,12 +185,12 @@ export const STRATEGIES: Record<string, PlanStrategy> = {
/** Throws USAGE if `plan` is not in the strategy table. */ /** Throws USAGE if `plan` is not in the strategy table. */
export function pickPlanStrategy(plan: string): PlanStrategy { export function pickPlanStrategy(plan: string): PlanStrategy {
const s = STRATEGIES[plan]; const strategy = STRATEGIES[plan];
if (!s) { if (!strategy) {
throw new BailianError( throw new BailianError(
`Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`, `Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`,
ExitCode.USAGE, ExitCode.USAGE,
); );
} }
return s; return strategy;
} }
@@ -4,12 +4,12 @@ import {
createFineTune, createFineTune,
getDataset, getDataset,
uploadDataset, uploadDataset,
validateDataset, detectModality,
getProfile,
fetchModelCapability, fetchModelCapability,
listSupportedTrainingTypes, listSupportedTrainingTypes,
preflightBatchSizeGate, preflightBatchSizeGate,
isTrainingTypeCli, isTrainingTypeCli,
toServerTrainingType,
TRAINING_TYPES_CLI, TRAINING_TYPES_CLI,
DEFAULT_TRAINING_TYPE, DEFAULT_TRAINING_TYPE,
formatIssue, formatIssue,
@@ -20,7 +20,8 @@ import {
type CreateFineTuneRequest, type CreateFineTuneRequest,
type FineTuneHyperParameters, type FineTuneHyperParameters,
type DatasetFile, type DatasetFile,
type DatasetSchema, type TrainingProfile,
type DataModality,
type FlagsDef, type FlagsDef,
} from "bailian-cli-core"; } from "bailian-cli-core";
import { existsSync, statSync } from "fs"; import { existsSync, statSync } from "fs";
@@ -77,7 +78,11 @@ async function analyzeDatasetTokens(
binName: string, binName: string,
raw: string, raw: string,
label: string, label: string,
schema?: DatasetSchema, profile: TrainingProfile,
_modality: DataModality,
model: string,
/** Pre-detected modality for a known path (avoids re-opening the file). */
knownModality?: { path: string; modality: DataModality },
): Promise<ResolvedDataset> { ): Promise<ResolvedDataset> {
const tokens = raw const tokens = raw
.split(",") .split(",")
@@ -109,11 +114,22 @@ async function analyzeDatasetTokens(
if (settings.dryRun) continue; if (settings.dryRun) continue;
// Local path → validate (same checks as `dataset upload`). Upload is // Detect modality per-file so each dataset is validated under the correct
// deferred to `uploadResolvedLocal` so the gate can run first. The schema // schema (e.g. a text JSONL and an audio ZIP in the same --datasets list
// (SFT vs DPO) is derived from --training-type so a DPO job validates the // are each validated with their own record schema). Reuse the caller's
// chosen/rejected preference pairs here, not on the platform. // pre-detected modality when available to avoid opening the same file
const result = await validateDataset(token, { schema }); // twice (matters for large ZIPs).
const tokenModality =
knownModality && knownModality.path === token
? knownModality.modality
: await detectModality(token);
// Local path → validate through the profile. The profile internally routes
// to the correct validator based on modality (detected from file content).
// Upload is deferred to `uploadResolvedLocal` so the gate can run first.
// `model` is forwarded for schema-agnostic cross-checks (e.g. the video
// validator verifies a kf2v model is paired with kf2v data).
const result = await profile.validate(token, tokenModality, { model });
if (!result.valid) { if (!result.valid) {
const lines = [ const lines = [
`Dataset validation failed for ${token}`, `Dataset validation failed for ${token}`,
@@ -306,20 +322,31 @@ export default defineCommand({
`Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`, `Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`,
); );
} }
// dpo / dpo-lora → "dpo" schema (strict chosen/rejected); cpt → "cpt"
// (raw {text} records); else ChatML ({messages}). // Profile: single source of truth for how this training type behaves
const datasetSchema: DatasetSchema = trainingType.startsWith("dpo") // (validation rules, hyper-parameters, gates, capability check).
? "dpo" const profile = getProfile(trainingType);
: trainingType === "cpt"
? "cpt" // Detect data modality from the first local file path in --datasets. This
: "chatml"; // drives the profile's internal branching (text vs audio vs image/video
// hyper-parameters, gate skips, capability check bypass). Falls back to
// "text" when no local file is available (file-id only). Image generation
// has a subtype "image-i2i" (first record has input_img) picked here.
const firstLocalPath = datasetsRaw
.split(",")
.map((token) => token.trim())
.find((token) => isLocalPath(token));
const modality: DataModality = firstLocalPath ? await detectModality(firstLocalPath) : "text";
const training = await analyzeDatasetTokens( const training = await analyzeDatasetTokens(
settings, settings,
identity.binName, identity.binName,
datasetsRaw, datasetsRaw,
"datasets", "datasets",
datasetSchema, profile,
modality,
model,
firstLocalPath ? { path: firstLocalPath, modality } : undefined,
); );
const trainingFileIds = training.fileIds; const trainingFileIds = training.fileIds;
@@ -329,7 +356,9 @@ export default defineCommand({
identity.binName, identity.binName,
flags.validations, flags.validations,
"validations", "validations",
datasetSchema, profile,
modality,
model,
) )
: undefined; : undefined;
const validationFileIds = validation?.fileIds; const validationFileIds = validation?.fileIds;
@@ -337,39 +366,52 @@ export default defineCommand({
const modelName = flags.modelName; const modelName = flags.modelName;
const suffix = flags.suffix; const suffix = flags.suffix;
// Hyper-parameters: inject n_epochs=3 default unless overridden. // Hyper-parameters: the profile resolves modality-specific defaults
const hp: FineTuneHyperParameters = {}; // (text: n_epochs/batch_size/learning_rate; audio: lm_max_epoch/fm_max_epoch/...).
hp.n_epochs = flags.nEpochs ?? 3; const hp = profile.resolveHyperParameters(
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate; modality,
if (flags.maxLength !== undefined) hp.max_length = flags.maxLength; flags as Record<string, unknown>,
) as FineTuneHyperParameters;
// batch_size: clamp to [8, 1024] (server hard constraint, undocumented). // Restore the batch-size clamping warning that was lost when the logic moved
// Surface the clamp on stderr instead of silently rewriting the user's // into profiles. The profile silently clamps to [8, 1024]; surface it here
// value — otherwise the submitted body would carry a number the user never // so the user has an audit trail. Skip modalities that bypass the batch_size
// typed, with no audit trail. (Range observed on common SFT / SFT-LoRA // gate (image/video): their batch_size is a fixed model-family default, not
// training types; some bases like qwen3.6-flash report a wider range, so // a clamp of the user's value, so the [8, 1024] "clamped" message would be
// the warning explicitly mentions "server range".) // self-contradictory (video uses 2/4) and misleading.
if (flags.batchSize !== undefined) { if (
flags.batchSize !== undefined &&
hp.batch_size !== undefined &&
!settings.quiet &&
!profile.shouldSkipGate("batch_size", modality)
) {
const requested = flags.batchSize; const requested = flags.batchSize;
let batchSize = requested; if (hp.batch_size !== requested) {
if (batchSize < 8) batchSize = 8;
if (batchSize > 1024) batchSize = 1024;
if (batchSize !== requested && !settings.quiet) {
process.stderr.write( process.stderr.write(
`warning: --batch-size ${requested} clamped to ${batchSize} ` + `warning: --batch-size ${requested} clamped to ${hp.batch_size} ` +
`(server range [8, 1024] for the common training types).\n`, `(server range [8, 1024] for the common training types).\n`,
); );
} }
hp.batch_size = batchSize; }
// For modalities that skip the batch_size gate (video: fixed 2/4 by model
// family), warn the user that their explicit --batch-size was discarded.
if (
flags.batchSize !== undefined &&
!settings.quiet &&
profile.shouldSkipGate("batch_size", modality)
) {
const requested = flags.batchSize;
if (hp.batch_size !== undefined && hp.batch_size !== requested) {
process.stderr.write(
`warning: --batch-size ${requested} ignored for ${modality} training ` +
`(model uses a fixed batch_size of ${hp.batch_size}).\n`,
);
}
} }
// Auto batch_size for small datasets: fetch first training file size. // Auto batch_size for small datasets — only for text data. Audio/image/video
// With default split=0.9, validation_set = 0.1 * rows. // profiles already set their own batch parameters.
// Platform default batch_size=16 needs rows > 160; batch_size=8 needs rows > 80. if (modality === "text" && hp.batch_size === undefined && !settings.dryRun) {
// Files < 100KB are conservatively estimated to have < 200 rows.
// If the first file was just uploaded we already hold its size; otherwise
// fall back to getDataset.
if (hp.batch_size === undefined && !settings.dryRun) {
let sizeBytes = training.firstSize ?? 0; let sizeBytes = training.firstSize ?? 0;
if (sizeBytes === 0) { if (sizeBytes === 0) {
try { try {
@@ -396,7 +438,11 @@ export default defineCommand({
// code as `validateDataset`) so the failure surfaces through the same // code as `validateDataset`) so the failure surfaces through the same
// `BailianError` + issue convention used by `dataset upload`/`validate`. // `BailianError` + issue convention used by `dataset upload`/`validate`.
// ExitCode.GENERAL matches the existing validation-failed exit code. // ExitCode.GENERAL matches the existing validation-failed exit code.
if (!settings.dryRun && training.recordCount !== undefined) { if (
!settings.dryRun &&
training.recordCount !== undefined &&
!profile.shouldSkipGate("batch_size", modality)
) {
// 16 is the platform default when neither the user nor the small-file // 16 is the platform default when neither the user nor the small-file
// auto-adjust set a batch_size (see the auto-adjust comment above). // auto-adjust set a batch_size (see the auto-adjust comment above).
const effectiveBatchSize = hp.batch_size ?? 16; const effectiveBatchSize = hp.batch_size ?? 16;
@@ -415,7 +461,7 @@ export default defineCommand({
// be trained against. listFoundationModels is a public API (no console // be trained against. listFoundationModels is a public API (no console
// login required); on lookup failure (network / 401 / etc.) we fall back // login required); on lookup failure (network / 401 / etc.) we fall back
// to letting the server decide rather than blocking the submit. // to letting the server decide rather than blocking the submit.
if (!settings.dryRun) { if (!settings.dryRun && !profile.shouldSkipCapabilityCheck(modality)) {
let capability: Awaited<ReturnType<typeof fetchModelCapability>> | undefined; let capability: Awaited<ReturnType<typeof fetchModelCapability>> | undefined;
try { try {
capability = await fetchModelCapability(settings, model); capability = await fetchModelCapability(settings, model);
@@ -453,8 +499,8 @@ export default defineCommand({
const body: CreateFineTuneRequest = { const body: CreateFineTuneRequest = {
model, model,
training_file_ids: trainingFileIds, training_file_ids: trainingFileIds,
// Map the CLI training type to the server value at the interface boundary. // Profile maps the CLI training type to the server value at the boundary.
training_type: toServerTrainingType(trainingType), training_type: profile.serverTrainingType,
hyper_parameters: hp, hyper_parameters: hp,
}; };
if (validationFileIds && validationFileIds.length > 0) { if (validationFileIds && validationFileIds.length > 0) {
+3 -1
View File
@@ -40,10 +40,12 @@
"check": "vp check" "check": "vp check"
}, },
"dependencies": { "dependencies": {
"yaml": "^2.8.3" "yaml": "^2.8.3",
"yauzl": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/yauzl": "catalog:",
"@typescript/native-preview": "7.0.0-dev.20260328.1", "@typescript/native-preview": "7.0.0-dev.20260328.1",
"typescript": "^6.0.2", "typescript": "^6.0.2",
"vite-plus": "catalog:" "vite-plus": "catalog:"
+2
View File
@@ -1,11 +1,13 @@
export * from "./types.ts"; export * from "./types.ts";
export * from "./api.ts"; export * from "./api.ts";
export { detectModality } from "./inspect.ts";
export { export {
validateDataset, validateDataset,
pickValidator, pickValidator,
registerValidator, registerValidator,
listSupportedFormats, listSupportedFormats,
MAX_DATASET_BYTES, MAX_DATASET_BYTES,
MAX_MEDIA_ZIP_BYTES,
parseDatasetSchemaFlag, parseDatasetSchemaFlag,
formatIssue, formatIssue,
} from "./validate/index.ts"; } from "./validate/index.ts";
+209
View File
@@ -0,0 +1,209 @@
/**
* Data inspector — lightweight content parser that determines the modality
* of a training data file.
*
* The inspector peeks at the first non-blank line of a JSONL file (or the
* `data.jsonl` manifest inside a ZIP) and inspects the record's fields to
* decide whether the data carries text, audio, image, or video samples.
*
* This is intentionally shallow — it reads at most one record — so it stays
* fast even on very large files. The full structural validation is the job
* of the format-specific validator (`jsonl.ts`, `zip.ts`), not this module.
*
* Routing in `create.ts`:
* 1. `--training-type` → Profile (via `getProfile`)
* 2. Profile.acceptedExtensions → match file extension
* 3. `detectModality(filePath)` → "text" | "audio" | "image" | "video"
* 4. Profile validates / resolves hyper-params using detected modality
*/
import { createReadStream } from "fs";
import { createInterface } from "readline";
import { extname } from "path";
import { BailianError } from "../errors/base.ts";
import { ExitCode } from "../errors/codes.ts";
import type { DataModality } from "../finetune/profiles/types.ts";
/**
* Inspect a file and return its data modality.
*
* `.jsonl` → read the first non-blank line, parse JSON, check fields.
* `.zip` → locate `data.jsonl` inside the archive, read its first line.
*
* Image data returns `"image"` (T2I) or `"image-i2i"` (I2I, first record
* has `input_img`). Callers that don't distinguish can normalise to `"image"`.
*
* Throws USAGE if the file extension is not `.jsonl` or `.zip`, or if the
* content cannot be parsed.
*/
export async function detectModality(filePath: string): Promise<DataModality> {
const ext = extname(filePath).toLowerCase();
if (ext === ".jsonl") return detectFromJsonl(filePath);
if (ext === ".zip") return detectFromZip(filePath);
throw new BailianError(
`Cannot inspect file with extension "${ext}". Expected .jsonl or .zip.`,
ExitCode.USAGE,
);
}
/**
* Read the first non-blank line of a JSONL file and determine the modality.
*/
async function detectFromJsonl(filePath: string): Promise<DataModality> {
const firstLine = await readFirstNonBlankLine(filePath);
if (!firstLine) {
throw new BailianError(
`JSONL file is empty or contains only blank lines: ${filePath}`,
ExitCode.USAGE,
);
}
const modality = classifyRecord(firstLine);
// JSONL files are always text data (chatml / dpo / cpt).
return modality === "unknown" ? "text" : modality;
}
/**
* Locate `data.jsonl` inside a ZIP archive, extract its first non-blank line,
* and determine the modality.
*
* Uses `yauzl` for streaming access — only the target entry is read, the rest
* of the archive is skipped.
*/
async function detectFromZip(filePath: string): Promise<DataModality> {
const firstLine = await readFirstLineFromZipEntry(filePath, "data.jsonl");
if (!firstLine) {
throw new BailianError(
`ZIP archive does not contain "data.jsonl" or it is empty: ${filePath}`,
ExitCode.USAGE,
`Audio training data must be a ZIP with data.jsonl at the root and a train/ subfolder.`,
);
}
const modality = classifyRecord(firstLine);
if (modality === "unknown") {
throw new BailianError(
`ZIP data.jsonl does not match any supported media format ` +
`(expected wav_fn / img_path / first_frame_path / video_path): ${filePath}`,
ExitCode.USAGE,
`ZIP archives are for audio/image/video training data. ` +
`For text data, use a .jsonl file instead.`,
);
}
return modality;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/** Classify a JSON record into a data modality based on its field names. */
function classifyRecord(line: string): DataModality | "unknown" {
let record: Record<string, unknown>;
try {
record = JSON.parse(line);
} catch {
throw new BailianError(
`Failed to parse first JSON record for modality detection: ${line.slice(0, 120)}`,
ExitCode.USAGE,
);
}
if (typeof record !== "object" || record === null || Array.isArray(record)) {
throw new BailianError(
`Expected a JSON object as the first record, got ${Array.isArray(record) ? "array" : typeof record}.`,
ExitCode.USAGE,
);
}
if ("wav_fn" in record) return "audio";
if ("img_path" in record) {
// Image generation: distinguish T2I (no input_img) from I2I (has input_img).
// The subtype drives hyper-parameter defaults (max_pixels 2k vs 1k).
return "input_img" in record ? "image-i2i" : "image";
}
if ("first_frame_path" in record || "video_path" in record) {
// Video generation (Wan i2v/kf2v): distinguish first-frame-only (i2v) from
// first+last-frame (kf2v, has last_frame_path). The subtype lets the
// profile cross-check the chosen --model against the data shape.
return "last_frame_path" in record ? "video-kf2v" : "video";
}
// No known media field found — caller decides how to handle.
return "unknown";
}
/** Read the first non-blank line from a file using a readline stream. */
function readFirstNonBlankLine(filePath: string): Promise<string | null> {
return new Promise((resolve, reject) => {
const stream = createReadStream(filePath, { encoding: "utf8" });
const rl = createInterface({ input: stream, crlfDelay: Infinity });
let found = false;
rl.on("line", (line) => {
if (found) return;
const trimmed = line.trim();
if (trimmed.length === 0) return;
found = true;
rl.close();
stream.destroy();
resolve(trimmed);
});
rl.on("close", () => {
if (!found) resolve(null);
});
rl.on("error", reject);
stream.on("error", reject);
});
}
/**
* Open a ZIP archive, locate the entry with the given name, and return the
* first non-blank line from its content. Returns `null` if the entry is not
* found or is empty.
*
* Delegates ZIP open/locate to the shared `openZipAndFindEntry` helper in
* `validate/zip.ts` — avoids duplicating yauzl boilerplate.
*/
function readFirstLineFromZipEntry(zipPath: string, entryName: string): Promise<string | null> {
return import("./validate/zip.ts")
.then(({ openZipAndFindEntry }) => openZipAndFindEntry(zipPath, entryName))
.then(({ entry, zipfile }) => {
return new Promise<string | null>((resolve, reject) => {
zipfile.openReadStream(entry, (streamErr, readStream) => {
if (streamErr || !readStream) {
zipfile.close();
reject(
new BailianError(
`Failed to read "${entryName}" from ZIP: ${streamErr?.message}`,
ExitCode.USAGE,
),
);
return;
}
const rl = createInterface({ input: readStream, crlfDelay: Infinity });
let found = false;
rl.on("line", (line) => {
if (found) return;
const trimmed = line.trim();
if (trimmed.length === 0) return;
found = true;
rl.close();
readStream.destroy();
zipfile.close();
resolve(trimmed);
});
rl.on("close", () => {
if (!found) {
zipfile.close();
resolve(null);
}
});
rl.on("error", (readError) => {
zipfile.close();
reject(readError);
});
});
});
})
.catch((error) => {
// openZipAndFindEntry rejects when the entry is not found — treat as null.
if (error instanceof Error && error.message.includes("not found in ZIP")) {
return null;
}
throw error;
});
}
+11 -3
View File
@@ -18,6 +18,13 @@ import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts
*/ */
export const MAX_DATASET_BYTES = 300 * 1024 * 1024; export const MAX_DATASET_BYTES = 300 * 1024 * 1024;
/**
* Image / video ZIP size cap 1 GB per the platform docs (vs 300 MB for
* text / audio). Used by `bl dataset upload` for media schemas and by the
* `sft-lora` training profile for image / video validation.
*/
export const MAX_MEDIA_ZIP_BYTES = 1024 * 1024 * 1024;
export interface PreflightResult { export interface PreflightResult {
bytes: number; bytes: number;
ext: string; ext: string;
@@ -75,11 +82,12 @@ export function emptyStats(): ValidationStats {
export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined { export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined {
if (value === undefined || value.trim() === "") return undefined; if (value === undefined || value.trim() === "") return undefined;
const v = value.trim(); const v = value.trim();
if (v === "chatml" || v === "dpo" || v === "cpt") return v; if (v === "chatml" || v === "dpo" || v === "cpt" || v === "tts" || v === "image" || v === "video")
return v;
throw new BailianError( throw new BailianError(
`Unsupported --schema "${value}". Supported: chatml, dpo, cpt.`, `Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image, video.`,
ExitCode.USAGE, ExitCode.USAGE,
`Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, else ChatML).`, `Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, first_frame_path/video_path → video, else ChatML).`,
); );
} }
+1 -1
View File
@@ -4,7 +4,7 @@ export {
registerValidator, registerValidator,
listSupportedFormats, listSupportedFormats,
} from "./registry.ts"; } from "./registry.ts";
export { MAX_DATASET_BYTES, parseDatasetSchemaFlag } from "./common.ts"; export { MAX_DATASET_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag } from "./common.ts";
export { formatIssue } from "./format.ts"; export { formatIssue } from "./format.ts";
export type { export type {
ValidatorSpec, ValidatorSpec,
@@ -16,10 +16,11 @@ import { extname } from "path";
import { BailianError } from "../../errors/base.ts"; import { BailianError } from "../../errors/base.ts";
import { ExitCode } from "../../errors/codes.ts"; import { ExitCode } from "../../errors/codes.ts";
import { jsonlValidator } from "./jsonl.ts"; import { jsonlValidator } from "./jsonl.ts";
import { zipValidator } from "./zip.ts";
import { preflight, MAX_DATASET_BYTES } from "./common.ts"; import { preflight, MAX_DATASET_BYTES } from "./common.ts";
import type { ValidatorSpec, ValidateOpts, ValidationResult } from "./types.ts"; import type { ValidatorSpec, ValidateOpts, ValidationResult } from "./types.ts";
const REGISTRY: ValidatorSpec[] = [jsonlValidator]; const REGISTRY: ValidatorSpec[] = [jsonlValidator, zipValidator];
/** Lookup the validator that handles a given file extension. */ /** Lookup the validator that handles a given file extension. */
export function pickValidator(filePath: string): ValidatorSpec { export function pickValidator(filePath: string): ValidatorSpec {
@@ -0,0 +1,177 @@
/**
* Image generation record schema Wan2.x fine-tuning.
*
* Two record flavours share the same schema:
* - **Text-to-image (T2I):** `{"prompt": "...", "img_path": "./x.png"}`
* - **Image-to-image (I2I):** `{"prompt": "...", "input_img": "./in.jpg", "img_path": "./out.jpg"}`
*
* The presence of `img_path` is the distinguishing field auto-detect picks
* this schema before the ChatML fallback. `input_img` is optional (I2I only).
*
* Image data lives in a ZIP with a flat layout (no `train/` subdirectory).
* File names must be ASCII-only per platform requirements.
*/
import { makeIssue } from "../common.ts";
import type { ValidationIssue } from "../types.ts";
import type { RecordSchemaSpec } from "./types.ts";
/** Accepted image file extensions (lower-case, with dot). */
export const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tiff"]);
/**
* Check that a path string ends with an accepted image extension.
* Returns the extension (lower-case) or an empty string.
*/
function imageExt(path: string): string {
const dot = path.lastIndexOf(".");
return dot >= 0 ? path.slice(dot).toLowerCase() : "";
}
/**
* Warn (non-error) when a filename contains non-ASCII characters.
* The platform requires English-only filenames.
*/
function asciiOnly(value: string): boolean {
return /^[\x20-\x7E]+$/.test(value);
}
function inspectImageRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
const out: ValidationIssue[] = [];
// --- prompt (required) ---
if (!("prompt" in record)) {
out.push(
makeIssue("error", "MISSING_PROMPT", `Required field "prompt" is missing.`, {
line: lineNo,
path: "prompt",
}),
);
} else {
const prompt = record.prompt;
if (typeof prompt !== "string") {
out.push(
makeIssue("error", "INVALID_PROMPT", `"prompt" must be a string (got ${typeof prompt}).`, {
line: lineNo,
path: "prompt",
}),
);
} else if (prompt.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_PROMPT", `"prompt" must not be empty / whitespace-only.`, {
line: lineNo,
path: "prompt",
}),
);
}
}
// --- img_path (required) ---
if (!("img_path" in record)) {
out.push(
makeIssue("error", "MISSING_IMG_PATH", `Required field "img_path" is missing.`, {
line: lineNo,
path: "img_path",
}),
);
} else {
const imgPath = record.img_path;
if (typeof imgPath !== "string") {
out.push(
makeIssue(
"error",
"INVALID_IMG_PATH",
`"img_path" must be a string (got ${typeof imgPath}).`,
{ line: lineNo, path: "img_path" },
),
);
} else if (imgPath.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_IMG_PATH", `"img_path" must not be empty.`, {
line: lineNo,
path: "img_path",
}),
);
} else {
const ext = imageExt(imgPath);
if (!IMAGE_EXTENSIONS.has(ext)) {
out.push(
makeIssue(
"warning",
"UNUSUAL_IMAGE_EXT",
`"img_path" points to a non-standard image extension "${ext || "(none)"}". ` +
`Expected one of: ${[...IMAGE_EXTENSIONS].join(", ")}.`,
{ line: lineNo, path: "img_path" },
),
);
}
if (!asciiOnly(imgPath)) {
out.push(
makeIssue(
"error",
"NON_ASCII_IMG_PATH",
`"img_path" must contain only ASCII characters (English filenames required). Got: "${imgPath}".`,
{ line: lineNo, path: "img_path" },
),
);
}
}
}
// --- input_img (optional — present only for I2I records) ---
if ("input_img" in record) {
const inputImg = record.input_img;
if (typeof inputImg !== "string") {
out.push(
makeIssue(
"error",
"INVALID_INPUT_IMG",
`"input_img" must be a string (got ${typeof inputImg}).`,
{ line: lineNo, path: "input_img" },
),
);
} else if (inputImg.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_INPUT_IMG", `"input_img" must not be empty.`, {
line: lineNo,
path: "input_img",
}),
);
} else {
const ext = imageExt(inputImg);
if (!IMAGE_EXTENSIONS.has(ext)) {
out.push(
makeIssue(
"warning",
"UNUSUAL_INPUT_IMG_EXT",
`"input_img" points to a non-standard image extension "${ext || "(none)"}". ` +
`Expected one of: ${[...IMAGE_EXTENSIONS].join(", ")}.`,
{ line: lineNo, path: "input_img" },
),
);
}
if (!asciiOnly(inputImg)) {
out.push(
makeIssue(
"error",
"NON_ASCII_INPUT_IMG",
`"input_img" must contain only ASCII characters (English filenames required). Got: "${inputImg}".`,
{ line: lineNo, path: "input_img" },
),
);
}
}
}
return out;
}
/**
* Image generation schema. Auto-detect: a record matches when it carries
* `img_path`. Placed before ChatML in the registry so image data is never
* misclassified.
*/
export const imageSchema: RecordSchemaSpec = {
name: "image",
detect: (record) => "img_path" in record,
inspect: inspectImageRecord,
};
@@ -16,11 +16,21 @@ import type { RecordSchemaSpec } from "./types.ts";
import { chatmlSchema } from "./chatml.ts"; import { chatmlSchema } from "./chatml.ts";
import { cptSchema } from "./cpt.ts"; import { cptSchema } from "./cpt.ts";
import { dpoSchema } from "./dpo.ts"; import { dpoSchema } from "./dpo.ts";
import { ttsSchema } from "./tts.ts";
import { imageSchema } from "./image.ts";
import { videoSchema } from "./video.ts";
// Order matters: DPO (chosen/rejected) and CPT (text) before ChatML (the // Order matters: TTS (wav_fn), image (img_path), video (first_frame_path/
// catch-all fallback). Each keys off a distinguishing field so the three // video_path), DPO (chosen/rejected) and CPT (text) before ChatML (the catch-
// partition cleanly — DPO never looks like CPT, etc. // all fallback). Each keys off a distinguishing field so they partition cleanly.
export const RECORD_SCHEMAS: RecordSchemaSpec[] = [dpoSchema, cptSchema, chatmlSchema]; export const RECORD_SCHEMAS: RecordSchemaSpec[] = [
ttsSchema,
imageSchema,
videoSchema,
dpoSchema,
cptSchema,
chatmlSchema,
];
/** /**
* Pick the right schema for a single parsed record. * Pick the right schema for a single parsed record.
@@ -0,0 +1,103 @@
/**
* TTS record schema `{"wav_fn": "train/xxx.wav", "text": "..."}`.
*
* Used for audio fine-tuning (e.g. CosyVoice v3 Flash). Each JSONL record
* inside the training data ZIP's `data.jsonl` maps a `.wav` file path to its
* transcript. The ZIP validator (`../zip.ts`) calls into this schema via the
* standard `jsonlValidator` pipeline the schema only owns per-record checks,
* the ZIP-level structural validation is separate.
*
* Auto-detect: a record matches when it carries `wav_fn` this is unique to
* audio training data and will never collide with ChatML/DPO/CPT.
*/
import { makeIssue } from "../common.ts";
import type { ValidationIssue } from "../types.ts";
import type { RecordSchemaSpec } from "./types.ts";
/** Expected audio file extensions (lower-case, with dot). */
const AUDIO_EXTENSIONS = new Set([".wav", ".mp3", ".flac", ".ogg", ".m4a"]);
function inspectTTSRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
const out: ValidationIssue[] = [];
// --- wav_fn ---
if (!("wav_fn" in record)) {
out.push(
makeIssue("error", "MISSING_WAV_FN", `Required field "wav_fn" is missing.`, {
line: lineNo,
path: "wav_fn",
}),
);
} else {
const wavFn = record.wav_fn;
if (typeof wavFn !== "string") {
out.push(
makeIssue("error", "INVALID_WAV_FN", `"wav_fn" must be a string (got ${typeof wavFn}).`, {
line: lineNo,
path: "wav_fn",
}),
);
} else if (wavFn.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_WAV_FN", `"wav_fn" must not be empty.`, {
line: lineNo,
path: "wav_fn",
}),
);
} else {
// Check that the path looks like it references an audio file.
const dotIndex = wavFn.lastIndexOf(".");
const ext = dotIndex >= 0 ? wavFn.slice(dotIndex).toLowerCase() : "";
if (!AUDIO_EXTENSIONS.has(ext)) {
out.push(
makeIssue(
"warning",
"UNUSUAL_AUDIO_EXT",
`"wav_fn" points to a non-standard audio extension "${ext || "(none)"}". ` +
`Expected one of: ${[...AUDIO_EXTENSIONS].join(", ")}.`,
{ line: lineNo, path: "wav_fn" },
),
);
}
}
}
// --- text ---
if (!("text" in record)) {
out.push(
makeIssue("error", "MISSING_TEXT", `Required field "text" is missing.`, {
line: lineNo,
path: "text",
}),
);
} else {
const text = record.text;
if (typeof text !== "string") {
out.push(
makeIssue("error", "INVALID_TEXT", `"text" must be a string (got ${typeof text}).`, {
line: lineNo,
path: "text",
}),
);
} else if (text.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_TEXT", `"text" must not be empty / whitespace-only.`, {
line: lineNo,
path: "text",
}),
);
}
}
return out;
}
/**
* TTS schema. Auto-detect: a record is treated as TTS when it carries `wav_fn`.
* Placed first in the registry so audio data is never misclassified as ChatML.
*/
export const ttsSchema: RecordSchemaSpec = {
name: "tts",
detect: (record) => "wav_fn" in record,
inspect: inspectTTSRecord,
};
@@ -0,0 +1,158 @@
/**
* Video generation record schema Wan i2v / kf2v fine-tuning.
*
* Two record flavours share the same schema:
* - **Image-to-video, first frame (i2v):**
* `{"prompt": "...", "first_frame_path": "image_1.jpg", "video_path": "video_1.mp4"}`
* - **Image-to-video, first+last frame (kf2v):**
* `{"prompt": "...", "first_frame_path": "image/x_first.jpg",
* "last_frame_path": "image/x_last.jpg", "video_path": "video/x.mp4"}`
*
* The presence of `first_frame_path` / `video_path` is the distinguishing
* signal auto-detect picks this schema before the ChatML fallback.
*
* `video_path` is OPTIONAL: validation-set records omit the target video (the
* platform generates preview videos from the first frame + prompt at each eval
* checkpoint), so the same schema validates both training and validation zips.
* `last_frame_path` is optional (kf2v only).
*
* Video data lives in a ZIP: i2v is flat, kf2v uses `image/` + `video/`
* subfolders. File names should be ASCII-only per platform requirements.
*/
import { makeIssue } from "../common.ts";
import type { ValidationIssue } from "../types.ts";
import type { RecordSchemaSpec } from "./types.ts";
/** Accepted image (frame) file extensions (lower-case, with dot). */
export const VIDEO_IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp"]);
/** Accepted video file extensions (lower-case, with dot). */
export const VIDEO_EXTENSIONS = new Set([".mp4", ".mov"]);
/** Return the lower-case extension of a path (with dot), or "". */
function pathExt(path: string): string {
const dot = path.lastIndexOf(".");
return dot >= 0 ? path.slice(dot).toLowerCase() : "";
}
/** The platform requires English-only (ASCII) file names. */
function asciiOnly(value: string): boolean {
return /^[\x20-\x7E]+$/.test(value);
}
/**
* Validate a required string path field, checking its extension against the
* accepted set and warning on non-ASCII names. Pushes issues into `out`.
*/
function checkPathField(
out: ValidationIssue[],
record: Record<string, unknown>,
field: string,
required: boolean,
accepted: Set<string>,
lineNo: number,
): void {
if (!(field in record)) {
if (required) {
out.push(
makeIssue("error", "MISSING_FIELD", `Required field "${field}" is missing.`, {
line: lineNo,
path: field,
}),
);
}
return;
}
const value = record[field];
if (typeof value !== "string") {
out.push(
makeIssue("error", "INVALID_FIELD", `"${field}" must be a string (got ${typeof value}).`, {
line: lineNo,
path: field,
}),
);
return;
}
if (value.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_FIELD", `"${field}" must not be empty.`, {
line: lineNo,
path: field,
}),
);
return;
}
const ext = pathExt(value);
if (!accepted.has(ext)) {
out.push(
makeIssue(
"warning",
"UNUSUAL_MEDIA_EXT",
`"${field}" points to a non-standard extension "${ext || "(none)"}". ` +
`Expected one of: ${[...accepted].join(", ")}.`,
{ line: lineNo, path: field },
),
);
}
if (!asciiOnly(value)) {
out.push(
makeIssue(
"error",
"NON_ASCII_PATH",
`"${field}" must contain only ASCII characters (English filenames required). Got: "${value}".`,
{ line: lineNo, path: field },
),
);
}
}
function inspectVideoRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
const out: ValidationIssue[] = [];
// --- prompt (required) ---
if (!("prompt" in record)) {
out.push(
makeIssue("error", "MISSING_PROMPT", `Required field "prompt" is missing.`, {
line: lineNo,
path: "prompt",
}),
);
} else {
const prompt = record.prompt;
if (typeof prompt !== "string") {
out.push(
makeIssue("error", "INVALID_PROMPT", `"prompt" must be a string (got ${typeof prompt}).`, {
line: lineNo,
path: "prompt",
}),
);
} else if (prompt.trim().length === 0) {
out.push(
makeIssue("error", "EMPTY_PROMPT", `"prompt" must not be empty / whitespace-only.`, {
line: lineNo,
path: "prompt",
}),
);
}
}
// --- first_frame_path (required) ---
checkPathField(out, record, "first_frame_path", true, VIDEO_IMAGE_EXTENSIONS, lineNo);
// --- last_frame_path (optional — kf2v only) ---
checkPathField(out, record, "last_frame_path", false, VIDEO_IMAGE_EXTENSIONS, lineNo);
// --- video_path (optional — training only; validation sets omit it) ---
checkPathField(out, record, "video_path", false, VIDEO_EXTENSIONS, lineNo);
return out;
}
/**
* Video generation schema. Auto-detect: a record matches when it carries
* `first_frame_path` or `video_path`. Placed before ChatML in the registry so
* video data is never misclassified. Distinct from image (`img_path`).
*/
export const videoSchema: RecordSchemaSpec = {
name: "video",
detect: (record) => "first_frame_path" in record || "video_path" in record,
inspect: inspectVideoRecord,
};
+8 -1
View File
@@ -32,10 +32,17 @@ export interface ValidateOpts {
* platform ten minutes in. * platform ten minutes in.
*/ */
schema?: DatasetSchema; schema?: DatasetSchema;
/**
* Model identifier (`--model`) forwarded for schema-agnostic cross-checks
* e.g. the video validator uses it to verify a `kf2v` model is paired with
* first+last-frame data. Optional: absent for bare file-id flows.
*/
model?: string;
} }
/** The schemas a `.jsonl` record can be validated against. */ /** The schemas a `.jsonl` record can be validated against. */
export type DatasetSchema = "chatml" | "dpo" | "cpt"; export type DatasetSchema = "chatml" | "dpo" | "cpt" | "tts" | "image" | "video";
export type ValidationSeverity = "error" | "warning"; export type ValidationSeverity = "error" | "warning";
+371
View File
@@ -0,0 +1,371 @@
/**
* ZIP validator audio / image / video training data archives.
*
* A training data ZIP must have:
* - `data.jsonl` at the root the manifest mapping media files to labels.
* - A `train/` subfolder (or media files at the root) referenced by the
* manifest entries.
*
* This validator owns the **ZIP-level structural checks** (entries present,
* references resolve). The **per-record JSONL content validation** is delegated
* to the existing `jsonlValidator` we extract `data.jsonl` to a temp file,
* run the full pipeline (quickScan + deepCheck + schema dispatch), and stitch
* the results together.
*
* The schema for `data.jsonl` records is passed via `opts.schema` (typically
* `"tts"` for audio). The profile layer decides which schema to use based on
* the detected modality this validator is schema-agnostic.
*/
import { createWriteStream, mkdirSync, rmSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { pipeline } from "stream/promises";
import { randomBytes } from "crypto";
import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts";
import { makeIssue } from "./common.ts";
import { jsonlValidator } from "./jsonl.ts";
import { IMAGE_EXTENSIONS } from "./schemas/image.ts";
/**
* Open a ZIP archive and locate a specific entry by name.
* Returns the entry and zipfile handle **caller must close the zipfile**.
* Normalises entry names (backslash forward-slash) and supports entries
* at the root or inside subdirectories (matches `name === targetName` or
* `name.endsWith("/${targetName}")`).
*
* Shared by `extractZipEntry` (this file) and `readFirstLineFromZipEntry`
* (inspect.ts) to avoid duplicating yauzl open/iterate/locate boilerplate.
*/
export function openZipAndFindEntry(
zipPath: string,
targetName: string,
): Promise<{ entry: import("yauzl").Entry; zipfile: import("yauzl").ZipFile }> {
return new Promise((resolve, reject) => {
import("yauzl")
.then((yauzl) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
zipfile.readEntry();
zipfile.on("entry", (entry) => {
const name = entry.fileName.replace(/\\/g, "/");
if (name === targetName || name.endsWith(`/${targetName}`)) {
resolve({ entry, zipfile });
} else {
zipfile.readEntry();
}
});
zipfile.on("end", () => {
zipfile.close();
reject(new Error(`Entry "${targetName}" not found in ZIP`));
});
zipfile.on("error", reject);
});
})
.catch(reject);
});
}
/**
* Collect all entry paths from a ZIP archive using yauzl.
* Returns normalised forward-slash paths.
*/
function collectZipEntries(zipPath: string): Promise<string[]> {
return new Promise((resolve, reject) => {
import("yauzl")
.then((yauzl) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
const entries: string[] = [];
zipfile.readEntry();
zipfile.on("entry", (entry) => {
entries.push(entry.fileName.replace(/\\/g, "/"));
zipfile.readEntry();
});
zipfile.on("end", () => {
zipfile.close();
resolve(entries);
});
zipfile.on("error", reject);
});
})
.catch(reject);
});
}
/**
* Extract a single entry from a ZIP archive to a destination path.
*/
async function extractZipEntry(
zipPath: string,
entryName: string,
destPath: string,
): Promise<void> {
const { entry, zipfile } = await openZipAndFindEntry(zipPath, entryName);
return new Promise((resolve, reject) => {
zipfile.openReadStream(entry, (streamErr, readStream) => {
if (streamErr || !readStream) {
zipfile.close();
reject(streamErr ?? new Error("Failed to open entry stream"));
return;
}
const writeStream = createWriteStream(destPath);
pipeline(readStream, writeStream)
.then(() => {
zipfile.close();
resolve();
})
.catch((pipelineError) => {
zipfile.close();
reject(pipelineError);
});
});
});
}
/**
* Read media file references from the first N records of a JSONL file to verify
* that referenced files exist inside the ZIP.
*
* Collects from all known schema fields: `wav_fn` (audio), `img_path` +
* `input_img` (image generation), `first_frame_path` / `last_frame_path` /
* `video_path` (video generation), `image_fn` / `video_fn` (legacy).
*/
async function collectMediaRefs(
jsonlPath: string,
maxLines = 100,
): Promise<{ refs: string[]; totalLines: number }> {
const { createReadStream } = await import("fs");
const { createInterface } = await import("readline");
const stream = createReadStream(jsonlPath, { encoding: "utf8" });
const rl = createInterface({ input: stream, crlfDelay: Infinity });
const refs: string[] = [];
let totalLines = 0;
for await (const raw of rl) {
totalLines++;
if (refs.length >= maxLines) continue;
const line = raw.trim();
if (line.length === 0) continue;
try {
const obj = JSON.parse(line) as Record<string, unknown>;
if (typeof obj.wav_fn === "string") refs.push(obj.wav_fn);
if (typeof obj.img_path === "string") refs.push(obj.img_path);
if (typeof obj.input_img === "string") refs.push(obj.input_img);
if (typeof obj.first_frame_path === "string") refs.push(obj.first_frame_path);
if (typeof obj.last_frame_path === "string") refs.push(obj.last_frame_path);
if (typeof obj.video_path === "string") refs.push(obj.video_path);
if (typeof obj.image_fn === "string") refs.push(obj.image_fn);
if (typeof obj.video_fn === "string") refs.push(obj.video_fn);
} catch {
// Parse errors are reported by the JSONL validator, not here.
}
}
return { refs, totalLines };
}
export const zipValidator: ValidatorSpec = {
format: "zip",
extensions: [".zip"],
async validate(filePath: string, opts: ValidateOpts): Promise<ValidationResult> {
const start = Date.now();
const errors: ValidationIssue[] = [];
const warnings: ValidationIssue[] = [];
// --- 1. Collect ZIP entries ---
let entries: string[];
try {
entries = await collectZipEntries(filePath);
} catch (openError) {
return {
valid: false,
format: "zip",
filePath,
errors: [
makeIssue(
"error",
"ZIP_OPEN_FAILED",
`Could not open ZIP archive: ${(openError as Error).message}`,
),
],
warnings: [],
stats: { durationMs: Date.now() - start },
};
}
if (entries.length === 0) {
return {
valid: false,
format: "zip",
filePath,
errors: [makeIssue("error", "ZIP_EMPTY", `ZIP archive contains no entries.`)],
warnings: [],
stats: { durationMs: Date.now() - start },
};
}
// --- 2. Check for data.jsonl ---
const hasDataJsonl = entries.some(
(entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"),
);
if (!hasDataJsonl) {
errors.push(
makeIssue(
"error",
"MISSING_DATA_JSONL",
`ZIP archive must contain "data.jsonl" at the root. ` +
`This file maps media files (e.g. .wav) to their labels.`,
),
);
}
// --- 3. Check for train/ directory (modality-aware) ---
// Audio TTS data uses a train/ subdirectory; image and video generation
// data do not (image is flat; video i2v is flat, kf2v uses image//video/).
// Only warn about missing train/ for audio (schema === "tts" or auto-detect
// when we don't know the schema yet).
const isImageSchema = opts.schema === "image";
const isVideoSchema = opts.schema === "video";
const hasTrainDir = entries.some((entry) => entry === "train/" || entry.startsWith("train/"));
// The train/ layout is an audio-TTS convention (media referenced as
// "train/xxx.wav"). It is only meaningful when the archive actually contains
// .wav files — image/video ZIPs (flat, or kf2v's image//video/ layout)
// legitimately have no train/ dir. Gating on .wav presence also fixes the
// auto-detect path (opts.schema undefined), where we would otherwise
// false-warn on every image/video archive.
const hasWavFiles = entries.some((entry) => entry.toLowerCase().endsWith(".wav"));
if (!hasTrainDir && !isImageSchema && !isVideoSchema && hasWavFiles) {
warnings.push(
makeIssue(
"warning",
"NO_TRAIN_DIR",
`No "train/" directory found in the ZIP. Media files are typically ` +
`placed under "train/" and referenced as "train/xxx.wav" in data.jsonl.`,
),
);
}
// --- 3b. Minimum image count check (image generation only) ---
// The platform requires at least 25 training images (50+ recommended).
if (isImageSchema) {
const MIN_IMAGES = 25;
const imageFiles = entries.filter((entry) => {
if (entry === "data.jsonl" || entry.endsWith("/data.jsonl")) return false;
if (entry.endsWith("/")) return false; // directory entries
const dot = entry.lastIndexOf(".");
const ext = dot >= 0 ? entry.slice(dot).toLowerCase() : "";
return IMAGE_EXTENSIONS.has(ext);
});
if (imageFiles.length < MIN_IMAGES) {
errors.push(
makeIssue(
"error",
"INSUFFICIENT_IMAGES",
`Found ${imageFiles.length} image(s) in ZIP, but image generation fine-tuning ` +
`requires at least ${MIN_IMAGES} images (50+ recommended).`,
),
);
}
}
// If data.jsonl is missing, we can't do JSONL content validation.
if (!hasDataJsonl) {
return {
valid: false,
format: "zip",
filePath,
errors,
warnings,
stats: { totalRecords: entries.length, durationMs: Date.now() - start },
};
}
// --- 4. Extract data.jsonl to a temp file and run jsonlValidator ---
const dataJsonlEntry = entries.find(
(entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"),
)!;
const tmpDir = join(tmpdir(), `bl-zip-${randomBytes(6).toString("hex")}`);
mkdirSync(tmpDir, { recursive: true });
const tmpJsonl = join(tmpDir, "data.jsonl");
try {
await extractZipEntry(filePath, dataJsonlEntry, tmpJsonl);
} catch (extractError) {
errors.push(
makeIssue(
"error",
"EXTRACT_FAILED",
`Failed to extract "data.jsonl" from ZIP: ${(extractError as Error).message}`,
),
);
rmSync(tmpDir, { recursive: true, force: true });
return {
valid: false,
format: "zip",
filePath,
errors,
warnings,
stats: { durationMs: Date.now() - start },
};
}
// Delegate JSONL content validation. The profile layer passes opts.schema
// (e.g. "tts") so the right record-schema spec is used.
const jsonlResult = await jsonlValidator.validate(tmpJsonl, opts);
errors.push(...jsonlResult.errors);
warnings.push(...jsonlResult.warnings);
// --- 5. Verify media file references (sample first 100 records) ---
if (jsonlResult.valid) {
const { refs } = await collectMediaRefs(tmpJsonl);
const entrySet = new Set(entries);
// Media paths in data.jsonl are relative to the manifest's location. Many
// official sample archives wrap everything in a single top-level folder
// (e.g. "wan-i2v-valid-dataset/data.jsonl" alongside
// "wan-i2v-valid-dataset/image_1.jpg"), so a bare "image_1.jpg" ref
// resolves against that folder, not the ZIP root. Derive the manifest's
// directory prefix and accept either the wrapped or root-relative form.
const slash = dataJsonlEntry.lastIndexOf("/");
const baseDir = slash >= 0 ? dataJsonlEntry.slice(0, slash + 1) : "";
const danglingRefs: string[] = [];
for (const ref of refs) {
// Normalise: some archives use "train/foo.wav", some use "./train/foo.wav".
const normalised = ref.replace(/^\.\//, "");
if (entrySet.has(normalised) || entrySet.has(baseDir + normalised)) continue;
danglingRefs.push(ref);
}
if (danglingRefs.length > 0) {
const shown = danglingRefs.slice(0, 5).join(", ");
const suffix = danglingRefs.length > 5 ? ` (and ${danglingRefs.length - 5} more)` : "";
errors.push(
makeIssue(
"error",
"DANGLING_MEDIA_REFS",
`${danglingRefs.length} media file(s) referenced in data.jsonl not found in ZIP: ${shown}${suffix}`,
),
);
}
}
// --- 6. Clean up ---
rmSync(tmpDir, { recursive: true, force: true });
return {
valid: errors.length === 0,
format: "zip",
filePath,
errors,
warnings,
stats: {
totalRecords: jsonlResult.stats.totalRecords ?? entries.length,
sampledRecords: jsonlResult.stats.sampledRecords,
durationMs: Date.now() - start,
},
};
},
};
+21 -2
View File
@@ -119,8 +119,8 @@ export interface CreateDeploymentRequest {
plan: string; plan: string;
/** Required by API even for token-billed (lora) plans where it is ignored — CLI injects 1. */ /** Required by API even for token-billed (lora) plans where it is ignored — CLI injects 1. */
capacity?: number; capacity?: number;
/** Optional template id for advanced configurations. */ /** Deploy spec id (e.g. "MU1", "dps-..."), sent as `deploy_spec` in POST body. */
template_id?: string; deploy_spec?: string;
/** /**
* PTU capacity (provisioned throughput limits). Only effective when * PTU capacity (provisioned throughput limits). Only effective when
* `plan === "ptu"`. The doc says this defaults to 10000/1000 when omitted, * `plan === "ptu"`. The doc says this defaults to 10000/1000 when omitted,
@@ -128,10 +128,29 @@ export interface CreateDeploymentRequest {
* info"), so the CLI treats it as required for ptu. * info"), so the CLI treats it as required for ptu.
*/ */
ptu_capacity?: PtuCapacity; ptu_capacity?: PtuCapacity;
/**
* AIGC generation config for fine-tuned Wan video (i2v/kf2v) LoRA deployments.
* Ignored by non-video plans. See `AigcConfig`.
*/
aigc_config?: AigcConfig;
/** Future-compat: arbitrary additional fields are forwarded as-is. */ /** Future-compat: arbitrary additional fields are forwarded as-is. */
[k: string]: unknown; [k: string]: unknown;
} }
/**
* AIGC generation config used when deploying fine-tuned Wan video (i2v/kf2v)
* LoRA models. Controls how prompts are applied at inference time:
* - use_input_prompt=false: ignore the caller's prompt, use the preset
* `prompt` template instead (the common LoRA case).
* - use_input_prompt=true: honor the caller's prompt.
* `lora_prompt_default` is the default trigger-word phrase appended for the LoRA.
*/
export interface AigcConfig {
use_input_prompt?: boolean;
prompt?: string;
lora_prompt_default?: string;
}
/** PTU throughput limits — only used when `plan === "ptu"`. */ /** PTU throughput limits — only used when `plan === "ptu"`. */
export interface PtuCapacity { export interface PtuCapacity {
/** Max input tokens per minute (all models). */ /** Max input tokens per minute (all models). */
-5
View File
@@ -57,11 +57,6 @@ export function isTrainingTypeCli(value: string): value is TrainingTypeCli {
return value in TRAINING_TYPE_MAP; return value in TRAINING_TYPE_MAP;
} }
/** Map a CLI training type to the server `training_type` for the request body. */
export function toServerTrainingType(value: TrainingTypeCli): string {
return TRAINING_TYPE_MAP[value].server;
}
/** The (method, variant) pair a CLI training type resolves to. */ /** The (method, variant) pair a CLI training type resolves to. */
export function trainingTypeMethodVariant(value: TrainingTypeCli): { export function trainingTypeMethodVariant(value: TrainingTypeCli): {
method: string; method: string;
+1
View File
@@ -2,3 +2,4 @@ export * from "./types.ts";
export * from "./api.ts"; export * from "./api.ts";
export * from "./capability.ts"; export * from "./capability.ts";
export * from "./preflight.ts"; export * from "./preflight.ts";
export * from "./profiles/index.ts";
@@ -0,0 +1,79 @@
/**
* Shared helpers for training profiles.
*
* Most text-based training types (sft, dpo, cpt, and their -lora variants)
* share the same hyper-parameter resolution logic: n_epochs defaults to 3,
* learning_rate and max_length are set only when explicitly provided, and
* batch_size is clamped to the server's [8, 1024] range. Extracting this
* into a single helper prevents drift when defaults or bounds change.
*/
import type { TrainingProfile, DataModality } from "./types.ts";
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
import type { DatasetSchema } from "../../dataset/validate/types.ts";
import { validateDataset } from "../../dataset/validate/registry.ts";
/**
* Resolve text-mode hyper-parameters from CLI flags.
*
* Shared by every profile's text branch (and by profiles that only support
* text). The audio branch in `sft-lora` uses its own `AUDIO_HYPER_PARAMS`.
*/
export function resolveTextHyperParameters(
flags: Record<string, unknown>,
): Record<string, unknown> {
const hp: Record<string, unknown> = {};
hp.n_epochs = flags.nEpochs !== undefined ? (flags.nEpochs as number) : 3;
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
if (flags.maxLength !== undefined) hp.max_length = flags.maxLength as number;
if (flags.batchSize !== undefined) {
const requested = flags.batchSize as number;
let batchSize = requested;
if (batchSize < 8) batchSize = 8;
if (batchSize > 1024) batchSize = 1024;
hp.batch_size = batchSize;
}
return hp;
}
/**
* Factory for text-only training profiles (sft, dpo, dpo-lora, cpt).
*
* These profiles are structurally identical they differ only in three
* string constants (CLI name, server name, record schema). Using a factory
* eliminates four near-identical files and prevents drift when the
* TrainingProfile interface changes.
*/
export function textProfile(
clientTrainingType: string,
serverTrainingType: string,
schema: DatasetSchema,
): TrainingProfile {
return {
clientTrainingType,
serverTrainingType,
acceptedExtensions: [".jsonl"],
async validate(
filePath: string,
_modality: DataModality,
opts: ValidateOpts,
): Promise<ValidationResult> {
return validateDataset(filePath, { ...opts, schema });
},
resolveHyperParameters(
_modality: DataModality,
flags: Record<string, unknown>,
): Record<string, unknown> {
return resolveTextHyperParameters(flags);
},
shouldSkipGate(_gate: string, _modality: DataModality): boolean {
return false;
},
shouldSkipCapabilityCheck(_modality: DataModality): boolean {
return false;
},
};
}
@@ -0,0 +1,7 @@
/**
* `cpt` profile Continual Pre-Training (full-parameter).
* Maps to the server's `cpt` training type. CPT record schema.
*/
import { textProfile } from "./common.ts";
export const cptProfile = textProfile("cpt", "cpt", "cpt");
@@ -0,0 +1,7 @@
/**
* `dpo-lora` profile LoRA variant of Direct Preference Optimization.
* Maps to the server's `dpo_lora` training type. DPO record schema.
*/
import { textProfile } from "./common.ts";
export const dpoLoraProfile = textProfile("dpo-lora", "dpo_lora", "dpo");
@@ -0,0 +1,7 @@
/**
* `dpo` profile Direct Preference Optimization (full-parameter).
* Maps to the server's `dpo_full` training type. DPO record schema.
*/
import { textProfile } from "./common.ts";
export const dpoProfile = textProfile("dpo", "dpo_full", "dpo");
@@ -0,0 +1,2 @@
export type { TrainingProfile, DataModality } from "./types.ts";
export { getProfile, listTrainingTypes } from "./registry.ts";
@@ -0,0 +1,50 @@
/**
* Training profile registry single point of truth for which training types
* the CLI supports.
*
* Routing: `--training-type <value>` exact match on `clientTrainingType`.
* Unknown values are rejected with a USAGE error listing all registered types.
*
* Adding a new training type:
* 1. Create `<name>.ts` exporting a `TrainingProfile` constant.
* 2. Import and append to `PROFILES`.
* That's it. `create.ts` never needs to change.
*/
import { BailianError } from "../../errors/base.ts";
import { ExitCode } from "../../errors/codes.ts";
import type { TrainingProfile } from "./types.ts";
import { sftProfile } from "./sft.ts";
import { sftLoraProfile } from "./sft-lora.ts";
import { dpoProfile } from "./dpo.ts";
import { dpoLoraProfile } from "./dpo-lora.ts";
import { cptProfile } from "./cpt.ts";
const PROFILES: TrainingProfile[] = [
sftProfile,
sftLoraProfile,
dpoProfile,
dpoLoraProfile,
cptProfile,
];
/** Look up a profile by its CLI training-type name. Throws USAGE if unknown. */
export function getProfile(clientTrainingType: string): TrainingProfile {
const p = PROFILES.find((p) => p.clientTrainingType === clientTrainingType);
if (!p) {
const known = PROFILES.map((p) => p.clientTrainingType).join(", ");
throw new BailianError(
`Unknown training type "${clientTrainingType}".`,
ExitCode.USAGE,
`Supported training types: ${known}.`,
);
}
return p;
}
/** All registered CLI training-type names (for help text / whitelisting). */
export function listTrainingTypes(): string[] {
return PROFILES.map((p) => p.clientTrainingType);
}
export type { TrainingProfile } from "./types.ts";
export type { DataModality } from "./types.ts";
@@ -0,0 +1,231 @@
/**
* `sft-lora` profile LoRA fine-tuning via the server's `efficient_sft`.
*
* Accepts `.jsonl` (text data with ChatML `{messages}` schema) and `.zip`
* (audio / image data with a `data.jsonl` manifest inside). The data
* inspector detects the modality from file content; this profile routes to the
* correct validator and assembles modality-specific hyper-parameters.
*
* Text, audio, and image all share the same server training type
* (`efficient_sft`) but differ in every other dimension: validation rules,
* hyper-parameters, and pre-flight gates. All branching is internal
* `create.ts` calls the uniform profile interface without knowing the modality.
*/
import type { TrainingProfile, DataModality } from "./types.ts";
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
import { validateDataset } from "../../dataset/validate/registry.ts";
import { makeIssue, MAX_MEDIA_ZIP_BYTES } from "../../dataset/validate/common.ts";
import { resolveTextHyperParameters } from "./common.ts";
/** Audio TTS hyper-parameter defaults (CosyVoice v3 Flash). */
const AUDIO_HYPER_PARAMS: Record<string, unknown> = {
lm_max_epoch: 60,
lm_step: 5,
lm_num: 3,
lm_batch_size: 1000,
fm_max_epoch: 100,
fm_step: 10,
fm_num: 3,
fm_batch_size: 2000,
};
/**
* Image generation hyper-parameter defaults (Wan2.x).
*
* The platform accepts these via `hyper_parameters` in the POST /fine-tunes
* body. `generation_type` controls `max_pixels` and `val_img_size` defaults:
* - t2i: "2k" (2048×2048)
* - i2i: "1k" (1024×1024)
*
* The default is "t2i". The generation type is auto-detected from data
* content: if the first JSONL record has `input_img`, it's I2I; otherwise T2I.
* The inspector returns `"image-i2i"` for I2I data, which this profile uses
* directly to pick the right hyper-parameter set.
* `split` (0.9) auto-splits training data into train/validation when no
* explicit validation_file_ids are provided.
*/
const IMAGE_HYPER_PARAMS_T2I: Record<string, unknown> = {
learning_rate: "3e-5",
max_steps: 800,
eval_steps: 200,
max_token_length: "1k",
gradient_clip: 0.5,
weight_decay: 0.02,
max_pixels: "2k",
val_img_size: "2k",
generation_type: "t2i",
lora_rank: 32,
save_total_limit: 10,
split: 0.9,
};
const IMAGE_HYPER_PARAMS_I2I: Record<string, unknown> = {
...IMAGE_HYPER_PARAMS_T2I,
max_pixels: "1k",
val_img_size: "1k",
generation_type: "i2i",
};
/**
* Image / video ZIP size cap shared constant from dataset/validate/common.ts.
*/
/**
* Video generation hyper-parameter defaults (Wan i2v / kf2v).
*
* Shared across all video models; `batch_size` and `max_pixels` differ by model
* family (resolved per model in `resolveHyperParameters`):
* - wan2.5 (e.g. wan2.5-i2v-preview): batch_size 2, max_pixels 36864
* - wan2.2 (i2v-flash / kf2v-flash): batch_size 4, max_pixels 262144
*
* `learning_rate` is a string to avoid JSON-number precision loss (consistent
* with the image defaults). `split` (0.9) + `max_split_val_dataset_sample`
* (5) drive the automatic train/validation split when no explicit
* validation_file_ids are provided.
*/
const VIDEO_HYPER_PARAMS_BASE: Record<string, unknown> = {
n_epochs: 400,
learning_rate: "2e-5",
split: 0.9,
max_split_val_dataset_sample: 5,
eval_epochs: 50,
save_total_limit: 10,
lora_rank: 32,
lora_alpha: 32,
};
/** True for any image modality (base or subtype). */
function isImage(modality: DataModality): boolean {
return modality === "image" || modality === "image-i2i";
}
/** True for any video modality (i2v base or kf2v subtype). */
function isVideo(modality: DataModality): boolean {
return modality === "video" || modality === "video-kf2v";
}
/** wan2.5 family uses smaller batch_size / max_pixels than wan2.2. */
function isWan25(model: string | undefined): boolean {
return typeof model === "string" && /wan2\.5/i.test(model);
}
export const sftLoraProfile: TrainingProfile = {
clientTrainingType: "sft-lora",
serverTrainingType: "efficient_sft",
acceptedExtensions: [".jsonl", ".zip"],
async validate(
filePath: string,
modality: DataModality,
opts: ValidateOpts,
): Promise<ValidationResult> {
if (modality === "audio") {
// ZIP validation: structure check + JSONL content via tts schema.
// The zip validator internally calls jsonlValidator for data.jsonl.
return validateDataset(filePath, { ...opts, schema: "tts" });
}
if (isImage(modality)) {
// Image generation: flat ZIP with data.jsonl + image files.
// The zip validator handles ≥25 image count + flat structure when
// schema is "image". Size cap is 1 GB (vs 300 MB for text/audio).
return validateDataset(filePath, {
...opts,
schema: "image",
maxBytes: MAX_MEDIA_ZIP_BYTES,
});
}
if (isVideo(modality)) {
// Video generation (Wan i2v/kf2v): ZIP with data.jsonl + frame images and
// (for training) target videos. i2v is flat; kf2v uses image//video/
// subfolders. 1 GB cap like image.
const result = await validateDataset(filePath, {
...opts,
schema: "video",
maxBytes: MAX_MEDIA_ZIP_BYTES,
});
// Model <-> data cross-check: a kf2v model needs first+last-frame data,
// an i2v model ignores last_frame_path. Only runs when we know the model
// (bare file-id flows pass no model). Detected subtype comes from the
// data inspector (video = i2v, video-kf2v = has last_frame_path).
if (typeof opts.model === "string" && opts.model.length > 0) {
const modelIsKf2v = /kf2v/i.test(opts.model);
const dataIsKf2v = modality === "video-kf2v";
if (modelIsKf2v && !dataIsKf2v) {
result.errors.push(
makeIssue(
"error",
"KF2V_DATA_MISMATCH",
`Model "${opts.model}" is a first+last-frame (kf2v) model but the data has ` +
`no "last_frame_path". kf2v training data must include a last frame per record.`,
),
);
} else if (!modelIsKf2v && dataIsKf2v) {
result.warnings.push(
makeIssue(
"warning",
"I2V_LAST_FRAME_IGNORED",
`Model "${opts.model}" is a first-frame (i2v) model but the data includes ` +
`"last_frame_path"; the last frame will be ignored during training.`,
),
);
}
}
result.valid = result.errors.length === 0;
return result;
}
// Text: standard JSONL validation with chatml schema.
return validateDataset(filePath, { ...opts, schema: "chatml" });
},
resolveHyperParameters(
modality: DataModality,
flags: Record<string, unknown>,
): Record<string, unknown> {
if (modality === "audio") {
// Audio: use TTS defaults, user flags override (MVP: all hardcoded).
return { ...AUDIO_HYPER_PARAMS };
}
if (isImage(modality)) {
// Image: modality "image-i2i" (auto-detected from data having input_img)
// uses I2I defaults; plain "image" uses T2I defaults.
const base = modality === "image-i2i" ? IMAGE_HYPER_PARAMS_I2I : IMAGE_HYPER_PARAMS_T2I;
const hp = { ...base };
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
return hp;
}
if (isVideo(modality)) {
// Video: shared defaults + model-family-specific batch_size / max_pixels.
// wan2.5 uses batch_size 2 / max_pixels 36864; wan2.2 uses 4 / 262144.
const wan25 = isWan25(flags.model as string | undefined);
const hp: Record<string, unknown> = {
...VIDEO_HYPER_PARAMS_BASE,
batch_size: wan25 ? 2 : 4,
max_pixels: wan25 ? 36864 : 262144,
};
// Optional overrides (no clamping — video batch_size is intentionally small).
if (flags.nEpochs !== undefined) hp.n_epochs = flags.nEpochs as number;
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
return hp;
}
// Text: existing hyper-parameter logic (n_epochs, batch_size, etc.).
return resolveTextHyperParameters(flags);
},
shouldSkipGate(gate: string, modality: DataModality): boolean {
if (modality === "audio" || isImage(modality) || isVideo(modality)) {
// Audio has no batch_size hyper-parameter; image uses max_steps; video
// datasets are intentionally small (batch_size 2/4). All skip the
// batch-size pre-flight gate to avoid false rejections.
if (gate === "batch_size") return true;
}
return false;
},
shouldSkipCapabilityCheck(modality: DataModality): boolean {
// Audio models (e.g. cosyvoice-v3-flash), image models
// (e.g. wan2.7-image-pro) and video models (e.g. wan2.5-i2v-preview) report
// supports.sft=false in listFoundationModels but the API accepts
// efficient_sft — skip to avoid false blocking.
return modality === "audio" || isImage(modality) || isVideo(modality);
},
};
@@ -0,0 +1,7 @@
/**
* `sft` profile full-parameter Supervised Fine-Tuning.
* Maps to the server's `sft` training type. ChatML record schema.
*/
import { textProfile } from "./common.ts";
export const sftProfile = textProfile("sft", "sft", "chatml");
@@ -0,0 +1,84 @@
/**
* Training profile single source of truth for "how a training type behaves".
*
* Each profile encapsulates everything the CLI needs to know about a specific
* `--training-type` value: what file formats it accepts, how to validate data,
* which hyper-parameters to use, which pre-flight gates to run, and whether
* the model-capability check should be skipped.
*
* Profiles are **decoupled from validators**: the profile declares which file
* extensions it accepts, and the data inspector (`dataset/inspect.ts`) detects
* the data modality by parsing the file content. The profile then routes to
* the appropriate validator internally `create.ts` never sees an if-else
* about modalities or file formats.
*
* Adding a new training type = one new profile file + one line in the registry.
* Nothing in `create.ts` needs to change.
*/
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
/**
* Data modality detected by the inspector from file content.
*
* The inspector peeks at the first record of a JSONL or the `data.jsonl` inside
* a ZIP to determine what kind of data the file carries. This is orthogonal to
* the training type `sft-lora` accepts both `.jsonl` (text) and `.zip`
* (audio / image / video).
*
* `"image-i2i"` is a subtype of `"image"` the first record contains an
* `input_img` field (image-to-image). `"video-kf2v"` is a subtype of `"video"`
* the first record contains a `last_frame_path` field (first+last-frame video,
* Wan kf2v). Profiles can branch on subtypes to adjust hyper-parameter defaults
* or cross-check the chosen `--model`. Callers that don't care about a subtype
* can normalise `"image-i2i"` `"image"` and `"video-kf2v"` `"video"`.
*/
export type DataModality = "text" | "audio" | "image" | "image-i2i" | "video" | "video-kf2v";
/**
* A training profile. One per `--training-type` CLI value.
*
* The profile owns all modality-specific branching internally callers
* (`create.ts`) interact with it through a uniform interface and never need
* to know whether the data is text, audio, or something else.
*/
export interface TrainingProfile {
/** CLI vocabulary: the value users pass to `--training-type`. */
clientTrainingType: string;
/** Server `training_type` for the POST /fine-tunes request body. */
serverTrainingType: string;
/** File extensions this profile accepts (lower-case, with dot). */
acceptedExtensions: string[];
/**
* Validate the training data file. The profile internally routes to the
* correct validator based on `modality` (detected by the data inspector).
*/
validate(filePath: string, modality: DataModality, opts: ValidateOpts): Promise<ValidationResult>;
/**
* Build the `hyper_parameters` object for the API request. Merges user-supplied
* flags with modality-specific defaults (e.g. audio TTS uses `lm_max_epoch` /
* `fm_max_epoch`, text SFT uses `n_epochs` / `batch_size`).
*/
resolveHyperParameters(
modality: DataModality,
flags: Record<string, unknown>,
): Record<string, unknown>;
/**
* Whether a specific pre-flight gate should be skipped for the given modality.
* Common gates: `"batch_size"` (samples must exceed batch_size), etc.
* Audio TTS skips the batch-size gate (no batch_size hyper-parameter).
*/
shouldSkipGate(gate: string, modality: DataModality): boolean;
/**
* Whether the model-capability pre-flight check should be skipped.
* Audio models (e.g. cosyvoice-v3-flash) report `supports.sft = false` in
* `listFoundationModels` but the API accepts `efficient_sft` the check
* would incorrectly block the submission.
*/
shouldSkipCapabilityCheck(modality: DataModality): boolean;
}
+4 -1
View File
@@ -169,10 +169,13 @@ describe("parseDatasetSchemaFlag", () => {
expect(parseDatasetSchemaFlag(" ")).toBeUndefined(); expect(parseDatasetSchemaFlag(" ")).toBeUndefined();
}); });
test("chatml / dpo / cpt pass through", () => { test("chatml / dpo / cpt / tts / image / video pass through", () => {
expect(parseDatasetSchemaFlag("chatml")).toBe("chatml"); expect(parseDatasetSchemaFlag("chatml")).toBe("chatml");
expect(parseDatasetSchemaFlag("dpo")).toBe("dpo"); expect(parseDatasetSchemaFlag("dpo")).toBe("dpo");
expect(parseDatasetSchemaFlag("cpt")).toBe("cpt"); expect(parseDatasetSchemaFlag("cpt")).toBe("cpt");
expect(parseDatasetSchemaFlag("tts")).toBe("tts");
expect(parseDatasetSchemaFlag("image")).toBe("image");
expect(parseDatasetSchemaFlag("video")).toBe("video");
expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo"); expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo");
}); });
+33 -3
View File
@@ -9,6 +9,9 @@ catalogs:
'@types/node': '@types/node':
specifier: ^24 specifier: ^24
version: 24.12.2 version: 24.12.2
'@types/yauzl':
specifier: ^3.4.0
version: 3.4.0
ajv: ajv:
specifier: ^8.20.0 specifier: ^8.20.0
version: 8.20.0 version: 8.20.0
@@ -27,6 +30,9 @@ catalogs:
yaml: yaml:
specifier: ^2.8.3 specifier: ^2.8.3
version: 2.8.3 version: 2.8.3
yauzl:
specifier: ^3.4.0
version: 3.4.0
overrides: overrides:
vite: npm:@voidzero-dev/vite-plus-core@latest vite: npm:@voidzero-dev/vite-plus-core@latest
@@ -119,10 +125,16 @@ importers:
yaml: yaml:
specifier: ^2.8.3 specifier: ^2.8.3
version: 2.8.3 version: 2.8.3
yauzl:
specifier: 'catalog:'
version: 3.4.0
devDependencies: devDependencies:
'@types/node': '@types/node':
specifier: 'catalog:' specifier: 'catalog:'
version: 24.12.2 version: 24.12.2
'@types/yauzl':
specifier: 'catalog:'
version: 3.4.0
'@typescript/native-preview': '@typescript/native-preview':
specifier: 7.0.0-dev.20260328.1 specifier: 7.0.0-dev.20260328.1
version: 7.0.0-dev.20260328.1 version: 7.0.0-dev.20260328.1
@@ -645,6 +657,9 @@ packages:
'@types/node@25.6.0': '@types/node@25.6.0':
resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}
'@types/yauzl@3.4.0':
resolution: {integrity: sha512-NRPn5w6h8dhcnmx3YIRQcqMywY/+nND/uOkJessedcrowO3C0AssHp3tMJpxKAwOhFOo0OV1y9VtsC5hbKKBAw==}
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1': '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
resolution: {integrity: sha512-BmJGDWC0bSQ2w5O/E+Mw9eTv9RklJ3vjshu7UdD92bUMxc4V4dkBhYj5r0qxbl4f+VFNX7fXvcDDI+9o+Kb6yw==} resolution: {integrity: sha512-BmJGDWC0bSQ2w5O/E+Mw9eTv9RklJ3vjshu7UdD92bUMxc4V4dkBhYj5r0qxbl4f+VFNX7fXvcDDI+9o+Kb6yw==}
cpu: [arm64] cpu: [arm64]
@@ -1021,6 +1036,9 @@ packages:
oxlint-tsgolint: oxlint-tsgolint:
optional: true optional: true
pend@1.2.0:
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
picocolors@1.1.1: picocolors@1.1.1:
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
@@ -1193,6 +1211,10 @@ packages:
engines: {node: '>= 14.6'} engines: {node: '>= 14.6'}
hasBin: true hasBin: true
yauzl@3.4.0:
resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==}
engines: {node: '>=12'}
snapshots: snapshots:
'@clack/core@0.3.5': '@clack/core@0.3.5':
@@ -1443,7 +1465,10 @@ snapshots:
'@types/node@25.6.0': '@types/node@25.6.0':
dependencies: dependencies:
undici-types: 7.19.2 undici-types: 7.19.2
optional: true
'@types/yauzl@3.4.0':
dependencies:
'@types/node': 25.6.0
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1': '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
optional: true optional: true
@@ -1781,6 +1806,8 @@ snapshots:
'@oxlint/binding-win32-x64-msvc': 1.63.0 '@oxlint/binding-win32-x64-msvc': 1.63.0
oxlint-tsgolint: 0.22.1 oxlint-tsgolint: 0.22.1
pend@1.2.0: {}
picocolors@1.1.1: {} picocolors@1.1.1: {}
picomatch@4.0.4: {} picomatch@4.0.4: {}
@@ -1874,8 +1901,7 @@ snapshots:
undici-types@7.16.0: {} undici-types@7.16.0: {}
undici-types@7.19.2: undici-types@7.19.2: {}
optional: true
undici@8.4.1: {} undici@8.4.1: {}
@@ -2016,3 +2042,7 @@ snapshots:
ws@8.20.0: {} ws@8.20.0: {}
yaml@2.8.3: {} yaml@2.8.3: {}
yauzl@3.4.0:
dependencies:
pend: 1.2.0
+2
View File
@@ -4,6 +4,7 @@ packages:
catalog: catalog:
"@types/node": ^24 "@types/node": ^24
"@types/yauzl": ^3.4.0
ajv: ^8.20.0 ajv: ^8.20.0
boxen: ^8.0.1 boxen: ^8.0.1
chalk: ^5.6.2 chalk: ^5.6.2
@@ -13,6 +14,7 @@ catalog:
vite-plus: latest vite-plus: latest
vitest: npm:@voidzero-dev/vite-plus-test@latest vitest: npm:@voidzero-dev/vite-plus-test@latest
yaml: ^2.8.3 yaml: ^2.8.3
yauzl: ^3.4.0
catalogMode: prefer catalogMode: prefer
overrides: overrides:
+59 -47
View File
@@ -7,13 +7,13 @@ Index: [index.md](index.md)
## Commands in this group ## Commands in this group
| Command | Description | | Command | Description |
| --------------------- | ---------------------------------------------------------- | | --------------------- | ------------------------------------------------------------------ |
| `bl dataset delete` | Delete a dataset file by ID | | `bl dataset delete` | Delete a dataset file by ID |
| `bl dataset get` | Get details of a single dataset file | | `bl dataset get` | Get details of a single dataset file |
| `bl dataset list` | List uploaded dataset files | | `bl dataset list` | List uploaded dataset files |
| `bl dataset upload` | Upload a dataset file (.jsonl) to Bailian | | `bl dataset upload` | Upload a dataset file (.jsonl or .zip) to Bailian |
| `bl dataset validate` | Locally validate a dataset file (.jsonl) without uploading | | `bl dataset validate` | Locally validate a dataset file (.jsonl or .zip) without uploading |
## Command details ## Command details
@@ -107,36 +107,36 @@ bl dataset list --output json
### `bl dataset upload` ### `bl dataset upload`
| Field | Value | | Field | Value |
| --------------- | -------------------------------------------------------------------------------------------------------------------- | | --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `dataset upload` | | **Name** | `dataset upload` |
| **Description** | Upload a dataset file (.jsonl) to Bailian | | **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt>] [--no-validate] [--full-validate]` | | **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
#### Flags #### Flags
| Flag | Type | Required | Description | | Flag | Type | Required | Description |
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------- | | ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local .jsonl dataset file (≤300MB) | | `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video) |
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | | `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record. | | `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | | `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | | `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--api-key <key>` | string | no | API key | | `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL | | `--base-url <url>` | string | no | API base URL |
#### Notes #### Notes
- Only .jsonl is supported in this release. Three record schemas are - Supports .jsonl (text) and .zip (audio/image/video archives with a
- recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...], - data.jsonl manifest). Six record schemas are recognized: chatml =
- chosen, rejected} where chosen/rejected are single assistant messages; - {messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};
- cpt = {text:"..."} (continual pre-training, raw text). With no --schema, - cpt = {text:"..."} (continual pre-training, raw text); tts =
- a record carrying chosen/rejected is validated as DPO, one with text (and - {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =
- no messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to - {img_path:"..."} (image generation); video = {first_frame_path:"...",
- require that shape on every record, or --schema chatml to ignore the - video_path:"..."} (video generation). With no --schema, a record
- preference / text fields. Other purposes may carry a different schema in - carrying wav_fn is validated as TTS, img_path as image, video_path /
- the future and would be served by a purpose-specific validator. - first_frame_path as video, chosen/rejected as DPO, text (no messages)
- The dataset upload cap is 300MB per file. - as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.
- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so - Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so
- the purpose tag is persisted (the DashScope-native /api/v1/files drops it). - the purpose tag is persisted (the DashScope-native /api/v1/files drops it).
@@ -154,6 +154,10 @@ bl dataset upload --file dpo.jsonl --schema dpo
bl dataset upload --file cpt.jsonl --schema cpt bl dataset upload --file cpt.jsonl --schema cpt
``` ```
```bash
bl dataset upload --file audio.zip --schema tts
```
```bash ```bash
bl dataset upload --file eval.jsonl --purpose evaluation bl dataset upload --file eval.jsonl --purpose evaluation
``` ```
@@ -168,31 +172,35 @@ bl dataset upload --file train.jsonl --no-validate
### `bl dataset validate` ### `bl dataset validate`
| Field | Value | | Field | Value |
| --------------- | ----------------------------------------------------------------------------------- | | --------------- | ------------------------------------------------------------------------------------------------------ |
| **Name** | `dataset validate` | | **Name** | `dataset validate` |
| **Description** | Locally validate a dataset file (.jsonl) without uploading | | **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt>]` | | **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
#### Flags #### Flags
| Flag | Type | Required | Description | | Flag | Type | Required | Description |
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------- | | ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local .jsonl dataset file | | `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | | `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record. | | `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
#### Notes #### Notes
- Default scan: every line gets a structural check, then ~160 lines (front 50, - Default scan: every line gets a structural check, then ~160 lines (front 50,
- evenly spaced 100, last 10) are JSON.parsed against the active schema. - evenly spaced 100, last 10) are JSON.parsed against the active schema.
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen, - Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
- rejected} where chosen/rejected are single assistant messages; cpt = - rejected}; cpt = {text:"..."} (continual pre-training, raw text);
- {text:"..."} (continual pre-training, raw text). With no --schema, a - tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
- record carrying chosen/rejected is validated as DPO, one with text (and no - image = {img_path:"..."} (image generation); video =
- messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to require - {first_frame_path:"...", video_path:"..."} (video generation). With no
- that shape on every record (strict), or --schema chatml to ignore the - --schema, a record carrying wav_fn is validated as TTS, img_path as
- preference / text fields. Use --full-validate to JSON.parse every line. - image, video_path / first_frame_path as video, chosen/rejected as DPO,
- text (no messages) as CPT, otherwise ChatML. Pass --schema to require a
- specific shape on every record. ZIP archives (.zip) are validated
- structurally (data.jsonl present, media references resolve) in addition
- to per-record content checks. Use --full-validate to JSON.parse every line.
#### Examples #### Examples
@@ -208,6 +216,10 @@ bl dataset validate --file dpo.jsonl --schema dpo
bl dataset validate --file cpt.jsonl --schema cpt bl dataset validate --file cpt.jsonl --schema cpt
``` ```
```bash
bl dataset validate --file audio.zip --schema tts
```
```bash ```bash
bl dataset validate --file eval.jsonl --full-validate bl dataset validate --file eval.jsonl --full-validate
``` ```
+27 -17
View File
@@ -25,23 +25,26 @@ Index: [index.md](index.md)
| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `deploy create` | | **Name** | `deploy create` |
| **Description** | Create a model deployment | | **Description** | Create a model deployment |
| **Usage** | `bl deploy create --model <model_name> --name <display_name> [--plan <plan>] [--template-id <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]` | | **Usage** | `bl deploy create --model <model_name> --name <display_name> [--plan <plan>] [--deploy-spec <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]` |
#### Flags #### Flags
| Flag | Type | Required | Description | | Flag | Type | Required | Description |
| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | | ----------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------ |
| `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) | | `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) |
| `--name <display_name>` | string | yes | Console display name for the deployment (required) | | `--name <display_name>` | string | yes | Console display name for the deployment (required) |
| `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | | `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu |
| `--template-id <id>` | string | no | Template id (only used by plan=mu; auto-picked if omitted) | | `--deploy-spec <id>` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) |
| `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | | `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) |
| `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | | `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) |
| `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) | | `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) |
| `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) | | `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) |
| `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) | | `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) |
| `--api-key <key>` | string | no | API key | | `--aigc-use-input-prompt <bool>` | boolean | no | Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template) |
| `--base-url <url>` | string | no | API base URL | | `--aigc-prompt <text>` | string | no | Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false |
| `--aigc-lora-prompt-default <text>` | string | no | Video LoRA (aigc_config): default trigger-word phrase for the LoRA |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
#### Notes #### Notes
@@ -49,12 +52,15 @@ Index: [index.md](index.md)
- For plan=ptu (Token-billed, provisioned throughput), --input-tpm and - For plan=ptu (Token-billed, provisioned throughput), --input-tpm and
- --output-tpm are required (the platform rejects creation without an - --output-tpm are required (the platform rejects creation without an
- explicit ptu_capacity despite the doc listing defaults). - explicit ptu_capacity despite the doc listing defaults).
- For plan=mu, `capacity`, `billing_method` and `template_id` are required. - For plan=mu, `capacity`, `billing_method` and `deploy_spec` are required.
- billing_method defaults to POST_PAY (only supported value); template_id - billing_method defaults to POST_PAY (only supported value); deploy_spec
- and capacity are auto-picked from GET /deployments/models when omitted. - and capacity are auto-picked from GET /deployments/models when omitted.
- Use `bl deploy models --source base` to inspect available templates. - Use `bl deploy models --source base` to inspect available templates.
- After creation, status starts at PENDING and transitions to RUNNING. - After creation, status starts at PENDING and transitions to RUNNING.
- Invoke the deployed model with: bl text chat --model <deployed_model> - Invoke the deployed model with: bl text chat --model <deployed_model>
- For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass
- --aigc-prompt / --aigc-lora-prompt-default (and optionally
- --aigc-use-input-prompt) to set the deployment's aigc_config.
- WARNING: --model is overloaded across commands and refers to DIFFERENT - WARNING: --model is overloaded across commands and refers to DIFFERENT
- values. `bl deploy create --model` takes the exported model_name (e.g. - values. `bl deploy create --model` takes the exported model_name (e.g.
- `qwen3-8b-ft-...`), but the create response also returns a `deployed_model` - `qwen3-8b-ft-...`), but the create response also returns a `deployed_model`
@@ -78,7 +84,11 @@ bl deploy create --model qwen3-8b --name my-qwen3-mu --plan mu
``` ```
```bash ```bash
bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --template-id MU1 --capacity 2 bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2
```
```bash
bl deploy create --model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."
``` ```
### `bl deploy delete` ### `bl deploy delete`
+2 -2
View File
@@ -22,8 +22,8 @@ Use this index for the full quick index and global flags.
| `bl dataset delete` | Delete a dataset file by ID | [dataset.md](dataset.md) | | `bl dataset delete` | Delete a dataset file by ID | [dataset.md](dataset.md) |
| `bl dataset get` | Get details of a single dataset file | [dataset.md](dataset.md) | | `bl dataset get` | Get details of a single dataset file | [dataset.md](dataset.md) |
| `bl dataset list` | List uploaded dataset files | [dataset.md](dataset.md) | | `bl dataset list` | List uploaded dataset files | [dataset.md](dataset.md) |
| `bl dataset upload` | Upload a dataset file (.jsonl) to Bailian | [dataset.md](dataset.md) | | `bl dataset upload` | Upload a dataset file (.jsonl or .zip) to Bailian | [dataset.md](dataset.md) |
| `bl dataset validate` | Locally validate a dataset file (.jsonl) without uploading | [dataset.md](dataset.md) | | `bl dataset validate` | Locally validate a dataset file (.jsonl or .zip) without uploading | [dataset.md](dataset.md) |
| `bl deploy create` | Create a model deployment | [deploy.md](deploy.md) | | `bl deploy create` | Create a model deployment | [deploy.md](deploy.md) |
| `bl deploy delete` | Delete a model deployment (must be STOPPED or FAILED) | [deploy.md](deploy.md) | | `bl deploy delete` | Delete a model deployment (must be STOPPED or FAILED) | [deploy.md](deploy.md) |
| `bl deploy get` | Get details of a single model deployment | [deploy.md](deploy.md) | | `bl deploy get` | Get details of a single model deployment | [deploy.md](deploy.md) |