mirror of
https://github.com/modelstudioai/cli.git
synced 2026-09-14 19:49:23 +08:00
feat: merge self-built-framework
This commit is contained in:
@@ -104,6 +104,17 @@ CLI 只为「自己能权威解释的错误」发出语义化信号,服务端的
|
|||||||
|
|
||||||
如果命令调用 Console Gateway,`defineCommand` 必须设置 `auth: "console"`。runtime 会基于 `CONSOLE_AUTH_FLAGS` 自动在 help 中展示 `--console-region`、`--console-site`、`--console-switch-agent`、`--workspace-id`,并由 `authStage` 解析/注入 console credential。命令不要重复声明这些凭证域 flag,也不要手动从 env/config 解析 token。
|
如果命令调用 Console Gateway,`defineCommand` 必须设置 `auth: "console"`。runtime 会基于 `CONSOLE_AUTH_FLAGS` 自动在 help 中展示 `--console-region`、`--console-site`、`--console-switch-agent`、`--workspace-id`,并由 `authStage` 解析/注入 console credential。命令不要重复声明这些凭证域 flag,也不要手动从 env/config 解析 token。
|
||||||
|
|
||||||
|
### 5. 禁止单字母变量命名
|
||||||
|
|
||||||
|
所有变量、参数、回调形参必须使用有语义的命名,不允许单字母(如 `i`、`m`、`p`、`t`、`e`、`s`)。具体表现:
|
||||||
|
|
||||||
|
- 回调参数: `.map((m) => ...)` → `.map((model) => ...)`, `.find((t) => ...)` → `.find((template) => ...)`
|
||||||
|
- catch 变量: `catch (e)` → `catch (error)`
|
||||||
|
- for-of 循环: `for (const i of items)` → `for (const item of items)`
|
||||||
|
- 临时变量: `const s = ...` → `const strategy = ...`
|
||||||
|
|
||||||
|
例外: 仅当作用域极小(≤3 行)且语义从上下文完全明确时,可使用 `k`/`v`(Object.entries 的 key/value)。
|
||||||
|
|
||||||
## 完成改动后的快速验证
|
## 完成改动后的快速验证
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -214,6 +214,52 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => {
|
|||||||
expect(data.action).toBe("dataset.upload");
|
expect(data.action).toBe("dataset.upload");
|
||||||
expect(data.schema).toBe("dpo");
|
expect(data.schema).toBe("dpo");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("dataset upload --schema image --no-validate --dry-run 采用 1GB 媒体上限", async () => {
|
||||||
|
// image/video schemas raise the upload cap to 1 GiB (vs 300 MB for text).
|
||||||
|
// --no-validate keeps this offline (the jsonl fixture is not a real zip).
|
||||||
|
const file = join(__dirname, ".dataset-valid.jsonl");
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"dataset",
|
||||||
|
"upload",
|
||||||
|
"--file",
|
||||||
|
file,
|
||||||
|
"--schema",
|
||||||
|
"image",
|
||||||
|
"--no-validate",
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stderr).toBe(0);
|
||||||
|
const data = parseStdoutJson<{ action: string; schema: string; max_bytes: number }>(stdout);
|
||||||
|
expect(data.action).toBe("dataset.upload");
|
||||||
|
expect(data.schema).toBe("image");
|
||||||
|
expect(data.max_bytes).toBe(1024 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["tts", "image", "video"])(
|
||||||
|
"dataset upload --dry-run 接受媒体 schema %s",
|
||||||
|
async (schema) => {
|
||||||
|
const file = join(__dirname, ".dataset-valid.jsonl");
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"dataset",
|
||||||
|
"upload",
|
||||||
|
"--file",
|
||||||
|
file,
|
||||||
|
"--schema",
|
||||||
|
schema,
|
||||||
|
"--no-validate",
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stderr).toBe(0);
|
||||||
|
const data = parseStdoutJson<{ action: string; schema: string }>(stdout);
|
||||||
|
expect(data.action).toBe("dataset.upload");
|
||||||
|
expect(data.schema).toBe(schema);
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => {
|
describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => {
|
||||||
|
|||||||
@@ -56,6 +56,98 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => {
|
|||||||
expect(data.body.capacity).toBe(1);
|
expect(data.body.capacity).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("deploy create --dry-run 组装视频 LoRA 的 aigc_config", async () => {
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"deploy",
|
||||||
|
"create",
|
||||||
|
"--model",
|
||||||
|
"wan2.5-i2v-preview-ft-xxx",
|
||||||
|
"--name",
|
||||||
|
"my-video-lora",
|
||||||
|
"--aigc-prompt",
|
||||||
|
"a cat surfing",
|
||||||
|
"--aigc-lora-prompt-default",
|
||||||
|
"trigger-word",
|
||||||
|
"--aigc-use-input-prompt",
|
||||||
|
"true",
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stderr).toBe(0);
|
||||||
|
const data = parseStdoutJson<{
|
||||||
|
action: string;
|
||||||
|
body: {
|
||||||
|
plan: string;
|
||||||
|
aigc_config?: {
|
||||||
|
use_input_prompt?: boolean;
|
||||||
|
prompt?: string;
|
||||||
|
lora_prompt_default?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}>(stdout);
|
||||||
|
expect(data.action).toBe("deploy.create");
|
||||||
|
expect(data.body.plan).toBe("lora");
|
||||||
|
expect(data.body.aigc_config).toEqual({
|
||||||
|
use_input_prompt: true,
|
||||||
|
prompt: "a cat surfing",
|
||||||
|
lora_prompt_default: "trigger-word",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("deploy create --aigc-* 仅对 plan=lora 有效(plan=ptu 时报错)", async () => {
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"deploy",
|
||||||
|
"create",
|
||||||
|
"--model",
|
||||||
|
"wan2.5-i2v-preview-ft-xxx",
|
||||||
|
"--name",
|
||||||
|
"my-video-lora",
|
||||||
|
"--plan",
|
||||||
|
"ptu",
|
||||||
|
"--input-tpm",
|
||||||
|
"10000",
|
||||||
|
"--output-tpm",
|
||||||
|
"1000",
|
||||||
|
"--aigc-prompt",
|
||||||
|
"a cat surfing",
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stdout + stderr).not.toBe(0);
|
||||||
|
expect(`${stdout}\n${stderr}`).toMatch(/--aigc-\* flags are only valid for plan=lora/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("deploy create --plan mu --deploy-spec --dry-run 透传 deploy_spec", async () => {
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"deploy",
|
||||||
|
"create",
|
||||||
|
"--model",
|
||||||
|
"qwen3-8b",
|
||||||
|
"--name",
|
||||||
|
"my-qwen3-mu",
|
||||||
|
"--plan",
|
||||||
|
"mu",
|
||||||
|
"--deploy-spec",
|
||||||
|
"MU1",
|
||||||
|
"--capacity",
|
||||||
|
"2",
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stderr).toBe(0);
|
||||||
|
const data = parseStdoutJson<{
|
||||||
|
action: string;
|
||||||
|
body: { plan: string; deploy_spec?: string; capacity?: number };
|
||||||
|
}>(stdout);
|
||||||
|
expect(data.action).toBe("deploy.create");
|
||||||
|
expect(data.body.plan).toBe("mu");
|
||||||
|
expect(data.body.deploy_spec).toBe("MU1");
|
||||||
|
expect(data.body.capacity).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
test("deploy scale --dry-run 转发 capacity", async () => {
|
test("deploy scale --dry-run 转发 capacity", async () => {
|
||||||
const { stdout, stderr, exitCode } = await runCli([
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
"deploy",
|
"deploy",
|
||||||
|
|||||||
@@ -114,6 +114,35 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test.each([
|
||||||
|
["sft", "sft"],
|
||||||
|
["sft-lora", "efficient_sft"],
|
||||||
|
["dpo", "dpo_full"],
|
||||||
|
["dpo-lora", "dpo_lora"],
|
||||||
|
["cpt", "cpt"],
|
||||||
|
])(
|
||||||
|
"finetune create --training-type %s 经 profile 映射为 server 类型 %s",
|
||||||
|
async (cliType, serverType) => {
|
||||||
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
|
"finetune",
|
||||||
|
"create",
|
||||||
|
"--model",
|
||||||
|
"qwen3-8b",
|
||||||
|
"--datasets",
|
||||||
|
"file-aaa",
|
||||||
|
"--training-type",
|
||||||
|
cliType,
|
||||||
|
"--dry-run",
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
]);
|
||||||
|
expect(exitCode, stderr).toBe(0);
|
||||||
|
const data = parseStdoutJson<{ action: string; body: { training_type: string } }>(stdout);
|
||||||
|
expect(data.action).toBe("finetune.create");
|
||||||
|
expect(data.body.training_type).toBe(serverType);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
test("finetune create --training-type 拒绝不支持的训练类型值", async () => {
|
test("finetune create --training-type 拒绝不支持的训练类型值", async () => {
|
||||||
const { stdout, stderr, exitCode } = await runCli([
|
const { stdout, stderr, exitCode } = await runCli([
|
||||||
"finetune",
|
"finetune",
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
parseDatasetSchemaFlag,
|
parseDatasetSchemaFlag,
|
||||||
formatIssue,
|
formatIssue,
|
||||||
MAX_DATASET_BYTES,
|
MAX_DATASET_BYTES,
|
||||||
|
MAX_MEDIA_ZIP_BYTES,
|
||||||
BailianError,
|
BailianError,
|
||||||
ExitCode,
|
ExitCode,
|
||||||
type DatasetFile,
|
type DatasetFile,
|
||||||
@@ -17,7 +18,7 @@ const UPLOAD_FLAGS = {
|
|||||||
file: {
|
file: {
|
||||||
type: "string",
|
type: "string",
|
||||||
valueHint: "<path>",
|
valueHint: "<path>",
|
||||||
description: "Local .jsonl dataset file (≤300MB)",
|
description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video)",
|
||||||
required: true,
|
required: true,
|
||||||
},
|
},
|
||||||
purpose: {
|
purpose: {
|
||||||
@@ -29,7 +30,7 @@ const UPLOAD_FLAGS = {
|
|||||||
type: "string",
|
type: "string",
|
||||||
valueHint: "<s>",
|
valueHint: "<s>",
|
||||||
description:
|
description:
|
||||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record.',
|
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
|
||||||
},
|
},
|
||||||
noValidate: {
|
noValidate: {
|
||||||
type: "switch",
|
type: "switch",
|
||||||
@@ -42,30 +43,31 @@ const UPLOAD_FLAGS = {
|
|||||||
} satisfies FlagsDef;
|
} satisfies FlagsDef;
|
||||||
|
|
||||||
export default defineCommand({
|
export default defineCommand({
|
||||||
description: "Upload a dataset file (.jsonl) to Bailian",
|
description: "Upload a dataset file (.jsonl or .zip) to Bailian",
|
||||||
auth: "apiKey",
|
auth: "apiKey",
|
||||||
usageArgs:
|
usageArgs:
|
||||||
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt>] [--no-validate] [--full-validate]",
|
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
|
||||||
flags: UPLOAD_FLAGS,
|
flags: UPLOAD_FLAGS,
|
||||||
exampleArgs: [
|
exampleArgs: [
|
||||||
"--file train.jsonl",
|
"--file train.jsonl",
|
||||||
"--file dpo.jsonl --schema dpo",
|
"--file dpo.jsonl --schema dpo",
|
||||||
"--file cpt.jsonl --schema cpt",
|
"--file cpt.jsonl --schema cpt",
|
||||||
|
"--file audio.zip --schema tts",
|
||||||
"--file eval.jsonl --purpose evaluation",
|
"--file eval.jsonl --purpose evaluation",
|
||||||
"--file train.jsonl --full-validate",
|
"--file train.jsonl --full-validate",
|
||||||
"--file train.jsonl --no-validate",
|
"--file train.jsonl --no-validate",
|
||||||
],
|
],
|
||||||
notes: [
|
notes: [
|
||||||
"Only .jsonl is supported in this release. Three record schemas are",
|
"Supports .jsonl (text) and .zip (audio/image/video archives with a",
|
||||||
"recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],",
|
"data.jsonl manifest). Six record schemas are recognized: chatml =",
|
||||||
"chosen, rejected} where chosen/rejected are single assistant messages;",
|
"{messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};",
|
||||||
'cpt = {text:"..."} (continual pre-training, raw text). With no --schema,',
|
'cpt = {text:"..."} (continual pre-training, raw text); tts =',
|
||||||
"a record carrying chosen/rejected is validated as DPO, one with text (and",
|
'{wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =',
|
||||||
"no messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to",
|
'{img_path:"..."} (image generation); video = {first_frame_path:"...",',
|
||||||
"require that shape on every record, or --schema chatml to ignore the",
|
'video_path:"..."} (video generation). With no --schema, a record',
|
||||||
"preference / text fields. Other purposes may carry a different schema in",
|
"carrying wav_fn is validated as TTS, img_path as image, video_path /",
|
||||||
"the future and would be served by a purpose-specific validator.",
|
"first_frame_path as video, chosen/rejected as DPO, text (no messages)",
|
||||||
"The dataset upload cap is 300MB per file.",
|
"as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.",
|
||||||
"Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so",
|
"Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so",
|
||||||
"the purpose tag is persisted (the DashScope-native /api/v1/files drops it).",
|
"the purpose tag is persisted (the DashScope-native /api/v1/files drops it).",
|
||||||
],
|
],
|
||||||
@@ -75,9 +77,16 @@ export default defineCommand({
|
|||||||
const purpose = flags.purpose || "fine-tune";
|
const purpose = flags.purpose || "fine-tune";
|
||||||
const schema = parseDatasetSchemaFlag(flags.schema);
|
const schema = parseDatasetSchemaFlag(flags.schema);
|
||||||
const format = detectOutputFormat(settings.output);
|
const format = detectOutputFormat(settings.output);
|
||||||
|
// Image / video schemas allow larger ZIPs (1 GB vs 300 MB for text).
|
||||||
|
const isMediaSchema = schema === "image" || schema === "video";
|
||||||
|
|
||||||
if (!flags.noValidate) {
|
if (!flags.noValidate) {
|
||||||
const result = await validateDataset(filePath, { fullValidate: flags.fullValidate, schema });
|
const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;
|
||||||
|
const result = await validateDataset(filePath, {
|
||||||
|
fullValidate: flags.fullValidate,
|
||||||
|
schema,
|
||||||
|
maxBytes,
|
||||||
|
});
|
||||||
if (!result.valid) {
|
if (!result.valid) {
|
||||||
const lines = [
|
const lines = [
|
||||||
`Dataset validation failed for ${filePath}`,
|
`Dataset validation failed for ${filePath}`,
|
||||||
@@ -112,7 +121,7 @@ export default defineCommand({
|
|||||||
action: "dataset.upload",
|
action: "dataset.upload",
|
||||||
file: filePath,
|
file: filePath,
|
||||||
purpose,
|
purpose,
|
||||||
max_bytes: MAX_DATASET_BYTES,
|
max_bytes: isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES,
|
||||||
validate: !flags.noValidate,
|
validate: !flags.noValidate,
|
||||||
schema: schema ?? "auto",
|
schema: schema ?? "auto",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const VALIDATE_FLAGS = {
|
|||||||
file: {
|
file: {
|
||||||
type: "string",
|
type: "string",
|
||||||
valueHint: "<path>",
|
valueHint: "<path>",
|
||||||
description: "Local .jsonl dataset file",
|
description: "Local dataset file (.jsonl or .zip)",
|
||||||
required: true,
|
required: true,
|
||||||
},
|
},
|
||||||
fullValidate: {
|
fullValidate: {
|
||||||
@@ -36,20 +36,21 @@ const VALIDATE_FLAGS = {
|
|||||||
type: "string",
|
type: "string",
|
||||||
valueHint: "<s>",
|
valueHint: "<s>",
|
||||||
description:
|
description:
|
||||||
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record.',
|
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
|
||||||
},
|
},
|
||||||
} satisfies FlagsDef;
|
} satisfies FlagsDef;
|
||||||
|
|
||||||
export default defineCommand({
|
export default defineCommand({
|
||||||
description: "Locally validate a dataset file (.jsonl) without uploading",
|
description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
|
||||||
// 纯本地校验,不触网、不需 API key(与 `pipeline validate` 一致)。
|
// 纯本地校验,不触网、不需 API key(与 `pipeline validate` 一致)。
|
||||||
auth: "none",
|
auth: "none",
|
||||||
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt>]",
|
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
|
||||||
flags: VALIDATE_FLAGS,
|
flags: VALIDATE_FLAGS,
|
||||||
exampleArgs: [
|
exampleArgs: [
|
||||||
"--file train.jsonl",
|
"--file train.jsonl",
|
||||||
"--file dpo.jsonl --schema dpo",
|
"--file dpo.jsonl --schema dpo",
|
||||||
"--file cpt.jsonl --schema cpt",
|
"--file cpt.jsonl --schema cpt",
|
||||||
|
"--file audio.zip --schema tts",
|
||||||
"--file eval.jsonl --full-validate",
|
"--file eval.jsonl --full-validate",
|
||||||
"--file train.jsonl --output json",
|
"--file train.jsonl --output json",
|
||||||
],
|
],
|
||||||
@@ -57,12 +58,16 @@ export default defineCommand({
|
|||||||
"Default scan: every line gets a structural check, then ~160 lines (front 50,",
|
"Default scan: every line gets a structural check, then ~160 lines (front 50,",
|
||||||
"evenly spaced 100, last 10) are JSON.parsed against the active schema.",
|
"evenly spaced 100, last 10) are JSON.parsed against the active schema.",
|
||||||
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
|
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
|
||||||
"rejected} where chosen/rejected are single assistant messages; cpt =",
|
'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
|
||||||
'{text:"..."} (continual pre-training, raw text). With no --schema, a',
|
'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
|
||||||
"record carrying chosen/rejected is validated as DPO, one with text (and no",
|
'image = {img_path:"..."} (image generation); video =',
|
||||||
"messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to require",
|
'{first_frame_path:"...", video_path:"..."} (video generation). With no',
|
||||||
"that shape on every record (strict), or --schema chatml to ignore the",
|
"--schema, a record carrying wav_fn is validated as TTS, img_path as",
|
||||||
"preference / text fields. Use --full-validate to JSON.parse every line.",
|
"image, video_path / first_frame_path as video, chosen/rejected as DPO,",
|
||||||
|
"text (no messages) as CPT, otherwise ChatML. Pass --schema to require a",
|
||||||
|
"specific shape on every record. ZIP archives (.zip) are validated",
|
||||||
|
"structurally (data.jsonl present, media references resolve) in addition",
|
||||||
|
"to per-record content checks. Use --full-validate to JSON.parse every line.",
|
||||||
],
|
],
|
||||||
async run(ctx) {
|
async run(ctx) {
|
||||||
const { settings, flags } = ctx;
|
const { settings, flags } = ctx;
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import {
|
|||||||
defineCommand,
|
defineCommand,
|
||||||
detectOutputFormat,
|
detectOutputFormat,
|
||||||
createDeployment,
|
createDeployment,
|
||||||
|
BailianError,
|
||||||
|
ExitCode,
|
||||||
type CreateDeploymentRequest,
|
type CreateDeploymentRequest,
|
||||||
type FlagsDef,
|
type FlagsDef,
|
||||||
} from "bailian-cli-core";
|
} from "bailian-cli-core";
|
||||||
@@ -26,10 +28,10 @@ const CREATE_FLAGS = {
|
|||||||
valueHint: "<plan>",
|
valueHint: "<plan>",
|
||||||
description: "Billing plan: lora (default, Token-billed) | ptu (Token-billed) | mu",
|
description: "Billing plan: lora (default, Token-billed) | ptu (Token-billed) | mu",
|
||||||
},
|
},
|
||||||
templateId: {
|
deploySpec: {
|
||||||
type: "string",
|
type: "string",
|
||||||
valueHint: "<id>",
|
valueHint: "<id>",
|
||||||
description: "Template id (only used by plan=mu; auto-picked if omitted)",
|
description: "Deploy spec (only used by plan=mu; auto-picked if omitted)",
|
||||||
},
|
},
|
||||||
capacity: {
|
capacity: {
|
||||||
type: "number",
|
type: "number",
|
||||||
@@ -56,6 +58,23 @@ const CREATE_FLAGS = {
|
|||||||
valueHint: "<n>",
|
valueHint: "<n>",
|
||||||
description: "PTU max thinking-output tokens/min (optional, some models)",
|
description: "PTU max thinking-output tokens/min (optional, some models)",
|
||||||
},
|
},
|
||||||
|
aigcUseInputPrompt: {
|
||||||
|
type: "boolean",
|
||||||
|
valueHint: "<bool>",
|
||||||
|
description:
|
||||||
|
"Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template)",
|
||||||
|
},
|
||||||
|
aigcPrompt: {
|
||||||
|
type: "string",
|
||||||
|
valueHint: "<text>",
|
||||||
|
description:
|
||||||
|
"Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false",
|
||||||
|
},
|
||||||
|
aigcLoraPromptDefault: {
|
||||||
|
type: "string",
|
||||||
|
valueHint: "<text>",
|
||||||
|
description: "Video LoRA (aigc_config): default trigger-word phrase for the LoRA",
|
||||||
|
},
|
||||||
} satisfies FlagsDef;
|
} satisfies FlagsDef;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -73,25 +92,29 @@ export default defineCommand({
|
|||||||
description: "Create a model deployment",
|
description: "Create a model deployment",
|
||||||
auth: "apiKey",
|
auth: "apiKey",
|
||||||
usageArgs:
|
usageArgs:
|
||||||
"--model <model_name> --name <display_name> [--plan <plan>] [--template-id <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]",
|
"--model <model_name> --name <display_name> [--plan <plan>] [--deploy-spec <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]",
|
||||||
flags: CREATE_FLAGS,
|
flags: CREATE_FLAGS,
|
||||||
exampleArgs: [
|
exampleArgs: [
|
||||||
"--model my-qwen-sft --name my-sft-test",
|
"--model my-qwen-sft --name my-sft-test",
|
||||||
"--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000",
|
"--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000",
|
||||||
"--model qwen3-8b --name my-qwen3-mu --plan mu",
|
"--model qwen3-8b --name my-qwen3-mu --plan mu",
|
||||||
"--model qwen3-8b --name my-qwen3 --plan mu --template-id MU1 --capacity 2",
|
"--model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2",
|
||||||
|
'--model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."',
|
||||||
],
|
],
|
||||||
notes: [
|
notes: [
|
||||||
"Plan defaults to `lora` (Token-billed). Pass --plan to override.",
|
"Plan defaults to `lora` (Token-billed). Pass --plan to override.",
|
||||||
"For plan=ptu (Token-billed, provisioned throughput), --input-tpm and",
|
"For plan=ptu (Token-billed, provisioned throughput), --input-tpm and",
|
||||||
"--output-tpm are required (the platform rejects creation without an",
|
"--output-tpm are required (the platform rejects creation without an",
|
||||||
"explicit ptu_capacity despite the doc listing defaults).",
|
"explicit ptu_capacity despite the doc listing defaults).",
|
||||||
"For plan=mu, `capacity`, `billing_method` and `template_id` are required.",
|
"For plan=mu, `capacity`, `billing_method` and `deploy_spec` are required.",
|
||||||
"billing_method defaults to POST_PAY (only supported value); template_id",
|
"billing_method defaults to POST_PAY (only supported value); deploy_spec",
|
||||||
"and capacity are auto-picked from GET /deployments/models when omitted.",
|
"and capacity are auto-picked from GET /deployments/models when omitted.",
|
||||||
"Use `bl deploy models --source base` to inspect available templates.",
|
"Use `bl deploy models --source base` to inspect available templates.",
|
||||||
"After creation, status starts at PENDING and transitions to RUNNING.",
|
"After creation, status starts at PENDING and transitions to RUNNING.",
|
||||||
"Invoke the deployed model with: bl text chat --model <deployed_model>",
|
"Invoke the deployed model with: bl text chat --model <deployed_model>",
|
||||||
|
"For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass",
|
||||||
|
"--aigc-prompt / --aigc-lora-prompt-default (and optionally",
|
||||||
|
"--aigc-use-input-prompt) to set the deployment's aigc_config.",
|
||||||
"WARNING: --model is overloaded across commands and refers to DIFFERENT",
|
"WARNING: --model is overloaded across commands and refers to DIFFERENT",
|
||||||
"values. `bl deploy create --model` takes the exported model_name (e.g.",
|
"values. `bl deploy create --model` takes the exported model_name (e.g.",
|
||||||
"`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`",
|
"`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`",
|
||||||
@@ -136,6 +159,33 @@ export default defineCommand({
|
|||||||
...resolved.body,
|
...resolved.body,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// AIGC config (fine-tuned Wan video LoRA deployments). Only valid for
|
||||||
|
// plan=lora — reject early for ptu/mu so the user gets a clear CLI error
|
||||||
|
// instead of an opaque server-side rejection.
|
||||||
|
const aigcUseInputPrompt = flags.aigcUseInputPrompt;
|
||||||
|
const aigcPrompt = flags.aigcPrompt;
|
||||||
|
const aigcLoraPromptDefault = flags.aigcLoraPromptDefault;
|
||||||
|
const hasAigcFlags =
|
||||||
|
aigcUseInputPrompt !== undefined ||
|
||||||
|
aigcPrompt !== undefined ||
|
||||||
|
aigcLoraPromptDefault !== undefined;
|
||||||
|
if (hasAigcFlags && plan !== "lora") {
|
||||||
|
throw new BailianError(
|
||||||
|
`--aigc-* flags are only valid for plan=lora (video LoRA deployments). Got plan=${plan}.`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (hasAigcFlags) {
|
||||||
|
const aigcConfig: Record<string, unknown> = {
|
||||||
|
use_input_prompt: aigcUseInputPrompt ?? false,
|
||||||
|
};
|
||||||
|
if (aigcPrompt !== undefined) aigcConfig.prompt = aigcPrompt;
|
||||||
|
if (aigcLoraPromptDefault !== undefined) {
|
||||||
|
aigcConfig.lora_prompt_default = aigcLoraPromptDefault;
|
||||||
|
}
|
||||||
|
body.aigc_config = aigcConfig;
|
||||||
|
}
|
||||||
|
|
||||||
if (settings.dryRun) {
|
if (settings.dryRun) {
|
||||||
emitResult({ action: "deploy.create", body }, format);
|
emitResult({ action: "deploy.create", body }, format);
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -59,8 +59,8 @@ export default defineCommand({
|
|||||||
ExitCode.USAGE,
|
ExitCode.USAGE,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (error) {
|
||||||
if (e instanceof BailianError) throw e;
|
if (error instanceof BailianError) throw error;
|
||||||
// If the get itself failed (e.g. not found), let the DELETE call surface the real error.
|
// If the get itself failed (e.g. not found), let the DELETE call surface the real error.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,13 +68,13 @@ export default defineCommand({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const headers = ["DEPLOYED_MODEL", "MODEL_NAME", "STATUS", "PLAN", "CAPACITY", "CREATED_AT"];
|
const headers = ["DEPLOYED_MODEL", "MODEL_NAME", "STATUS", "PLAN", "CAPACITY", "CREATED_AT"];
|
||||||
const rows = items.map((i) => [
|
const rows = items.map((item) => [
|
||||||
i.deployed_model,
|
item.deployed_model,
|
||||||
i.model_name,
|
item.model_name,
|
||||||
i.status,
|
item.status,
|
||||||
i.plan,
|
item.plan,
|
||||||
i.capacity,
|
item.capacity,
|
||||||
i.created_at,
|
item.created_at,
|
||||||
]);
|
]);
|
||||||
for (const line of formatTable(headers, rows)) emitBare(line);
|
for (const line of formatTable(headers, rows)) emitBare(line);
|
||||||
if (total !== undefined) emitBare(`\nTotal: ${total}`);
|
if (total !== undefined) emitBare(`\nTotal: ${total}`);
|
||||||
|
|||||||
@@ -76,44 +76,44 @@ export default defineCommand({
|
|||||||
// downstream tooling can drive `bl deploy create --template-id <…>` without
|
// downstream tooling can drive `bl deploy create --template-id <…>` without
|
||||||
// a second round-trip. For text: keep the compact one-line summary.
|
// a second round-trip. For text: keep the compact one-line summary.
|
||||||
if (format === "json") {
|
if (format === "json") {
|
||||||
const items = models.map((m) => {
|
const items = models.map((model) => {
|
||||||
const out: Record<string, unknown> = {
|
const out: Record<string, unknown> = {
|
||||||
model_name: m.model_name ?? "",
|
model_name: model.model_name ?? "",
|
||||||
};
|
};
|
||||||
if (m.base_model) out.base_model = m.base_model;
|
if (model.base_model) out.base_model = model.base_model;
|
||||||
if (m.model_source) out.model_source = m.model_source;
|
if (model.model_source) out.model_source = model.model_source;
|
||||||
if (m.supported_plans && m.supported_plans.length > 0) {
|
if (model.supported_plans && model.supported_plans.length > 0) {
|
||||||
out.supported_plans = m.supported_plans;
|
out.supported_plans = model.supported_plans;
|
||||||
}
|
}
|
||||||
if (m.plans && m.plans.length > 0) {
|
if (model.plans && model.plans.length > 0) {
|
||||||
out.plans = m.plans.map((p) => {
|
out.plans = model.plans.map((plan) => {
|
||||||
const planEntry: Record<string, unknown> = { plan: p.plan ?? "" };
|
const planEntry: Record<string, unknown> = { plan: plan.plan ?? "" };
|
||||||
if (p.cu_specs && p.cu_specs.length > 0) {
|
if (plan.cu_specs && plan.cu_specs.length > 0) {
|
||||||
planEntry.cu_specs = p.cu_specs;
|
planEntry.cu_specs = plan.cu_specs;
|
||||||
}
|
}
|
||||||
if (p.templates && p.templates.length > 0) {
|
if (plan.templates && plan.templates.length > 0) {
|
||||||
// Pull the top 6 fields most useful for `bl deploy create`.
|
// Pull the top 6 fields most useful for `bl deploy create`.
|
||||||
// Drop noisy/redundant: template_source, template_type,
|
// Drop noisy/redundant: template_source, template_type,
|
||||||
// template_version, deploy_spec (typically == template_id).
|
// template_version, deploy_spec (typically == template_id).
|
||||||
planEntry.templates = p.templates.map((t) => {
|
planEntry.templates = plan.templates.map((template) => {
|
||||||
const tpl: Record<string, unknown> = {};
|
const tpl: Record<string, unknown> = {};
|
||||||
if (t.template_id) tpl.template_id = t.template_id;
|
if (template.template_id) tpl.template_id = template.template_id;
|
||||||
if (t.template_name) tpl.template_name = t.template_name;
|
if (template.template_name) tpl.template_name = template.template_name;
|
||||||
if (t.charge_type) tpl.charge_type = t.charge_type;
|
if (template.charge_type) tpl.charge_type = template.charge_type;
|
||||||
// Flatten roles.unified for the common COUPLED case.
|
// Flatten roles.unified for the common COUPLED case.
|
||||||
const unified = t.roles?.unified;
|
const unified = template.roles?.unified;
|
||||||
if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec;
|
if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec;
|
||||||
if (unified?.capacity_unit_per_instance !== undefined)
|
if (unified?.capacity_unit_per_instance !== undefined)
|
||||||
tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance;
|
tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance;
|
||||||
// Preserve split-role configs (SEPERATED) as-is so callers
|
// Preserve split-role configs (SEPERATED) as-is so callers
|
||||||
// can still drive prefill/decode sizing.
|
// can still drive prefill/decode sizing.
|
||||||
if (t.roles?.prefill || t.roles?.decode) {
|
if (template.roles?.prefill || template.roles?.decode) {
|
||||||
tpl.roles = {
|
tpl.roles = {
|
||||||
prefill: t.roles?.prefill,
|
prefill: template.roles?.prefill,
|
||||||
decode: t.roles?.decode,
|
decode: template.roles?.decode,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (t.template_desc) tpl.template_desc = t.template_desc;
|
if (template.template_desc) tpl.template_desc = template.template_desc;
|
||||||
return tpl;
|
return tpl;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -127,19 +127,19 @@ export default defineCommand({
|
|||||||
}
|
}
|
||||||
|
|
||||||
// text / quiet — keep the compact single-line summary table.
|
// text / quiet — keep the compact single-line summary table.
|
||||||
const textItems = models.map((m) => {
|
const textItems = models.map((model) => {
|
||||||
let plansSummary = "";
|
let plansSummary = "";
|
||||||
if (m.supported_plans && m.supported_plans.length > 0) {
|
if (model.supported_plans && model.supported_plans.length > 0) {
|
||||||
plansSummary = m.supported_plans.join(",");
|
plansSummary = model.supported_plans.join(",");
|
||||||
} else if (m.plans && m.plans.length > 0) {
|
} else if (model.plans && model.plans.length > 0) {
|
||||||
plansSummary = m.plans
|
plansSummary = model.plans
|
||||||
.map((p) => {
|
.map((plan) => {
|
||||||
const planName = p.plan ?? "?";
|
const planName = plan.plan ?? "?";
|
||||||
if (p.templates && p.templates.length > 0) {
|
if (plan.templates && plan.templates.length > 0) {
|
||||||
return `${planName}(${p.templates.length}t)`;
|
return `${planName}(${plan.templates.length}t)`;
|
||||||
}
|
}
|
||||||
if (p.cu_specs && p.cu_specs.length > 0) {
|
if (plan.cu_specs && plan.cu_specs.length > 0) {
|
||||||
return `${planName}(${p.cu_specs.join("/")})`;
|
return `${planName}(${plan.cu_specs.join("/")})`;
|
||||||
}
|
}
|
||||||
return planName;
|
return planName;
|
||||||
})
|
})
|
||||||
@@ -148,9 +148,9 @@ export default defineCommand({
|
|||||||
plansSummary = "-";
|
plansSummary = "-";
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
model_name: m.model_name ?? "",
|
model_name: model.model_name ?? "",
|
||||||
base_model: m.base_model ?? "",
|
base_model: model.base_model ?? "",
|
||||||
source: m.model_source ?? "",
|
source: model.model_source ?? "",
|
||||||
plans: plansSummary,
|
plans: plansSummary,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -160,7 +160,12 @@ export default defineCommand({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const headers = ["MODEL_NAME", "BASE_MODEL", "SOURCE", "PLANS"];
|
const headers = ["MODEL_NAME", "BASE_MODEL", "SOURCE", "PLANS"];
|
||||||
const rows = textItems.map((i) => [i.model_name, i.base_model, i.source, i.plans]);
|
const rows = textItems.map((item) => [
|
||||||
|
item.model_name,
|
||||||
|
item.base_model,
|
||||||
|
item.source,
|
||||||
|
item.plans,
|
||||||
|
]);
|
||||||
for (const line of formatTable(headers, rows)) emitBare(line);
|
for (const line of formatTable(headers, rows)) emitBare(line);
|
||||||
if (total !== undefined) emitBare(`\nTotal: ${total}`);
|
if (total !== undefined) emitBare(`\nTotal: ${total}`);
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import { listDeployableModels, BailianError, ExitCode, type Client } from "baili
|
|||||||
/** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */
|
/** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */
|
||||||
export interface CreatePlanFlags {
|
export interface CreatePlanFlags {
|
||||||
plan?: string;
|
plan?: string;
|
||||||
templateId?: string;
|
deploySpec?: string;
|
||||||
capacity?: number;
|
capacity?: number;
|
||||||
billingMethod?: string;
|
billingMethod?: string;
|
||||||
inputTpm?: number;
|
inputTpm?: number;
|
||||||
@@ -101,15 +101,15 @@ const ptuStrategy: PlanStrategy = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `mu` (model-unit-billed). `capacity`, `billing_method` and `template_id` are
|
* `mu` (model-unit-billed). `capacity`, `billing_method` and `deploy_spec` are
|
||||||
* all required by the API but every one has a CLI-side default:
|
* all required by the API but every one has a CLI-side default:
|
||||||
* - billing_method defaults to POST_PAY (the only supported value).
|
* - billing_method defaults to POST_PAY (the only supported value).
|
||||||
* - template_id auto-picks from GET /deployments/models — the one whose
|
* - deploy_spec auto-picks from GET /deployments/models — the one whose
|
||||||
* `charge_type` matches `billing_method`, else the first available.
|
* `charge_type` matches `billing_method`, else the first available.
|
||||||
* - capacity defaults to the template's `capacity_unit_per_instance` (the
|
* - capacity defaults to the template's `capacity_unit_per_instance` (the
|
||||||
* smallest valid multiple of base_capacity).
|
* smallest valid multiple of base_capacity).
|
||||||
*
|
*
|
||||||
* The catalog lookup is skipped when `--template-id` is supplied explicitly:
|
* The catalog lookup is skipped when `--deploy-spec` is supplied explicitly:
|
||||||
* fine-tuned custom models may not appear in the `source=base` catalog, and
|
* fine-tuned custom models may not appear in the `source=base` catalog, and
|
||||||
* forcing the lookup would otherwise raise a spurious "no template" error.
|
* forcing the lookup would otherwise raise a spurious "no template" error.
|
||||||
* It is also skipped in dry-run mode to keep `--dry-run` side-effect-free.
|
* It is also skipped in dry-run mode to keep `--dry-run` side-effect-free.
|
||||||
@@ -121,15 +121,15 @@ const muStrategy: PlanStrategy = {
|
|||||||
},
|
},
|
||||||
async resolve(ctx: PlanContext): Promise<PlanResolved> {
|
async resolve(ctx: PlanContext): Promise<PlanResolved> {
|
||||||
const billingMethod = ctx.flags.billingMethod || "POST_PAY";
|
const billingMethod = ctx.flags.billingMethod || "POST_PAY";
|
||||||
let templateId = ctx.flags.templateId;
|
let deploySpec = ctx.flags.deploySpec;
|
||||||
let capacity = ctx.flags.capacity;
|
let capacity = ctx.flags.capacity;
|
||||||
|
|
||||||
if (!ctx.dryRun && !templateId) {
|
if (!ctx.dryRun && !deploySpec) {
|
||||||
const noTemplateError = () =>
|
const noTemplateError = () =>
|
||||||
new BailianError(
|
new BailianError(
|
||||||
`No mu-plan template found for model "${ctx.model}". ` +
|
`No mu-plan template found for model "${ctx.model}". ` +
|
||||||
`Run \`${ctx.binName} deploy models --source base\` to inspect available models, ` +
|
`Run \`${ctx.binName} deploy models --source base\` to inspect available models, ` +
|
||||||
`or pass --template-id explicitly.`,
|
`or pass --deploy-spec explicitly.`,
|
||||||
ExitCode.USAGE,
|
ExitCode.USAGE,
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
@@ -139,23 +139,24 @@ const muStrategy: PlanStrategy = {
|
|||||||
version: "v1.0",
|
version: "v1.0",
|
||||||
});
|
});
|
||||||
const payload = resp.output ?? resp.data;
|
const payload = resp.output ?? resp.data;
|
||||||
const target = (payload?.models ?? []).find((m) => m.model_name === ctx.model);
|
const target = (payload?.models ?? []).find((model) => model.model_name === ctx.model);
|
||||||
const muPlan = target?.plans?.find((p) => p.plan === "mu");
|
const muPlan = target?.plans?.find((plan) => plan.plan === "mu");
|
||||||
const templates = muPlan?.templates ?? [];
|
const templates = muPlan?.templates ?? [];
|
||||||
if (templates.length === 0) throw noTemplateError();
|
if (templates.length === 0) throw noTemplateError();
|
||||||
// POST_PAY → post_paid template; fall back to the first available.
|
// POST_PAY → post_paid template; fall back to the first available.
|
||||||
const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid";
|
const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid";
|
||||||
const picked = templates.find((t) => t.charge_type === wantChargeType) ?? templates[0];
|
const picked =
|
||||||
if (!picked?.template_id) throw noTemplateError();
|
templates.find((template) => template.charge_type === wantChargeType) ?? templates[0];
|
||||||
templateId = picked.template_id;
|
if (!picked?.deploy_spec && !picked?.template_id) throw noTemplateError();
|
||||||
|
deploySpec = picked.deploy_spec ?? picked.template_id;
|
||||||
if (capacity === undefined) {
|
if (capacity === undefined) {
|
||||||
capacity = picked.roles?.unified?.capacity_unit_per_instance ?? 1;
|
capacity = picked.roles?.unified?.capacity_unit_per_instance ?? 1;
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (error) {
|
||||||
if (e instanceof BailianError) throw e;
|
if (error instanceof BailianError) throw error;
|
||||||
throw new BailianError(
|
throw new BailianError(
|
||||||
`Failed to auto-pick template for plan=mu: ${(e as Error).message}. ` +
|
`Failed to auto-pick template for plan=mu: ${(error as Error).message}. ` +
|
||||||
`Pass --template-id explicitly.`,
|
`Pass --deploy-spec explicitly.`,
|
||||||
ExitCode.USAGE,
|
ExitCode.USAGE,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -165,7 +166,7 @@ const muStrategy: PlanStrategy = {
|
|||||||
capacity: capacity ?? 1,
|
capacity: capacity ?? 1,
|
||||||
billing_method: billingMethod,
|
billing_method: billingMethod,
|
||||||
};
|
};
|
||||||
if (templateId) body.template_id = templateId;
|
if (deploySpec) body.deploy_spec = deploySpec;
|
||||||
return { body };
|
return { body };
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -184,12 +185,12 @@ export const STRATEGIES: Record<string, PlanStrategy> = {
|
|||||||
|
|
||||||
/** Throws USAGE if `plan` is not in the strategy table. */
|
/** Throws USAGE if `plan` is not in the strategy table. */
|
||||||
export function pickPlanStrategy(plan: string): PlanStrategy {
|
export function pickPlanStrategy(plan: string): PlanStrategy {
|
||||||
const s = STRATEGIES[plan];
|
const strategy = STRATEGIES[plan];
|
||||||
if (!s) {
|
if (!strategy) {
|
||||||
throw new BailianError(
|
throw new BailianError(
|
||||||
`Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`,
|
`Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`,
|
||||||
ExitCode.USAGE,
|
ExitCode.USAGE,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return s;
|
return strategy;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,12 +4,12 @@ import {
|
|||||||
createFineTune,
|
createFineTune,
|
||||||
getDataset,
|
getDataset,
|
||||||
uploadDataset,
|
uploadDataset,
|
||||||
validateDataset,
|
detectModality,
|
||||||
|
getProfile,
|
||||||
fetchModelCapability,
|
fetchModelCapability,
|
||||||
listSupportedTrainingTypes,
|
listSupportedTrainingTypes,
|
||||||
preflightBatchSizeGate,
|
preflightBatchSizeGate,
|
||||||
isTrainingTypeCli,
|
isTrainingTypeCli,
|
||||||
toServerTrainingType,
|
|
||||||
TRAINING_TYPES_CLI,
|
TRAINING_TYPES_CLI,
|
||||||
DEFAULT_TRAINING_TYPE,
|
DEFAULT_TRAINING_TYPE,
|
||||||
formatIssue,
|
formatIssue,
|
||||||
@@ -20,7 +20,8 @@ import {
|
|||||||
type CreateFineTuneRequest,
|
type CreateFineTuneRequest,
|
||||||
type FineTuneHyperParameters,
|
type FineTuneHyperParameters,
|
||||||
type DatasetFile,
|
type DatasetFile,
|
||||||
type DatasetSchema,
|
type TrainingProfile,
|
||||||
|
type DataModality,
|
||||||
type FlagsDef,
|
type FlagsDef,
|
||||||
} from "bailian-cli-core";
|
} from "bailian-cli-core";
|
||||||
import { existsSync, statSync } from "fs";
|
import { existsSync, statSync } from "fs";
|
||||||
@@ -77,7 +78,11 @@ async function analyzeDatasetTokens(
|
|||||||
binName: string,
|
binName: string,
|
||||||
raw: string,
|
raw: string,
|
||||||
label: string,
|
label: string,
|
||||||
schema?: DatasetSchema,
|
profile: TrainingProfile,
|
||||||
|
_modality: DataModality,
|
||||||
|
model: string,
|
||||||
|
/** Pre-detected modality for a known path (avoids re-opening the file). */
|
||||||
|
knownModality?: { path: string; modality: DataModality },
|
||||||
): Promise<ResolvedDataset> {
|
): Promise<ResolvedDataset> {
|
||||||
const tokens = raw
|
const tokens = raw
|
||||||
.split(",")
|
.split(",")
|
||||||
@@ -109,11 +114,22 @@ async function analyzeDatasetTokens(
|
|||||||
|
|
||||||
if (settings.dryRun) continue;
|
if (settings.dryRun) continue;
|
||||||
|
|
||||||
// Local path → validate (same checks as `dataset upload`). Upload is
|
// Detect modality per-file so each dataset is validated under the correct
|
||||||
// deferred to `uploadResolvedLocal` so the gate can run first. The schema
|
// schema (e.g. a text JSONL and an audio ZIP in the same --datasets list
|
||||||
// (SFT vs DPO) is derived from --training-type so a DPO job validates the
|
// are each validated with their own record schema). Reuse the caller's
|
||||||
// chosen/rejected preference pairs here, not on the platform.
|
// pre-detected modality when available to avoid opening the same file
|
||||||
const result = await validateDataset(token, { schema });
|
// twice (matters for large ZIPs).
|
||||||
|
const tokenModality =
|
||||||
|
knownModality && knownModality.path === token
|
||||||
|
? knownModality.modality
|
||||||
|
: await detectModality(token);
|
||||||
|
|
||||||
|
// Local path → validate through the profile. The profile internally routes
|
||||||
|
// to the correct validator based on modality (detected from file content).
|
||||||
|
// Upload is deferred to `uploadResolvedLocal` so the gate can run first.
|
||||||
|
// `model` is forwarded for schema-agnostic cross-checks (e.g. the video
|
||||||
|
// validator verifies a kf2v model is paired with kf2v data).
|
||||||
|
const result = await profile.validate(token, tokenModality, { model });
|
||||||
if (!result.valid) {
|
if (!result.valid) {
|
||||||
const lines = [
|
const lines = [
|
||||||
`Dataset validation failed for ${token}`,
|
`Dataset validation failed for ${token}`,
|
||||||
@@ -306,20 +322,31 @@ export default defineCommand({
|
|||||||
`Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`,
|
`Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// dpo / dpo-lora → "dpo" schema (strict chosen/rejected); cpt → "cpt"
|
|
||||||
// (raw {text} records); else ChatML ({messages}).
|
// Profile: single source of truth for how this training type behaves
|
||||||
const datasetSchema: DatasetSchema = trainingType.startsWith("dpo")
|
// (validation rules, hyper-parameters, gates, capability check).
|
||||||
? "dpo"
|
const profile = getProfile(trainingType);
|
||||||
: trainingType === "cpt"
|
|
||||||
? "cpt"
|
// Detect data modality from the first local file path in --datasets. This
|
||||||
: "chatml";
|
// drives the profile's internal branching (text vs audio vs image/video
|
||||||
|
// hyper-parameters, gate skips, capability check bypass). Falls back to
|
||||||
|
// "text" when no local file is available (file-id only). Image generation
|
||||||
|
// has a subtype "image-i2i" (first record has input_img) picked here.
|
||||||
|
const firstLocalPath = datasetsRaw
|
||||||
|
.split(",")
|
||||||
|
.map((token) => token.trim())
|
||||||
|
.find((token) => isLocalPath(token));
|
||||||
|
const modality: DataModality = firstLocalPath ? await detectModality(firstLocalPath) : "text";
|
||||||
|
|
||||||
const training = await analyzeDatasetTokens(
|
const training = await analyzeDatasetTokens(
|
||||||
settings,
|
settings,
|
||||||
identity.binName,
|
identity.binName,
|
||||||
datasetsRaw,
|
datasetsRaw,
|
||||||
"datasets",
|
"datasets",
|
||||||
datasetSchema,
|
profile,
|
||||||
|
modality,
|
||||||
|
model,
|
||||||
|
firstLocalPath ? { path: firstLocalPath, modality } : undefined,
|
||||||
);
|
);
|
||||||
const trainingFileIds = training.fileIds;
|
const trainingFileIds = training.fileIds;
|
||||||
|
|
||||||
@@ -329,7 +356,9 @@ export default defineCommand({
|
|||||||
identity.binName,
|
identity.binName,
|
||||||
flags.validations,
|
flags.validations,
|
||||||
"validations",
|
"validations",
|
||||||
datasetSchema,
|
profile,
|
||||||
|
modality,
|
||||||
|
model,
|
||||||
)
|
)
|
||||||
: undefined;
|
: undefined;
|
||||||
const validationFileIds = validation?.fileIds;
|
const validationFileIds = validation?.fileIds;
|
||||||
@@ -337,39 +366,52 @@ export default defineCommand({
|
|||||||
const modelName = flags.modelName;
|
const modelName = flags.modelName;
|
||||||
const suffix = flags.suffix;
|
const suffix = flags.suffix;
|
||||||
|
|
||||||
// Hyper-parameters: inject n_epochs=3 default unless overridden.
|
// Hyper-parameters: the profile resolves modality-specific defaults
|
||||||
const hp: FineTuneHyperParameters = {};
|
// (text: n_epochs/batch_size/learning_rate; audio: lm_max_epoch/fm_max_epoch/...).
|
||||||
hp.n_epochs = flags.nEpochs ?? 3;
|
const hp = profile.resolveHyperParameters(
|
||||||
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate;
|
modality,
|
||||||
if (flags.maxLength !== undefined) hp.max_length = flags.maxLength;
|
flags as Record<string, unknown>,
|
||||||
|
) as FineTuneHyperParameters;
|
||||||
|
|
||||||
// batch_size: clamp to [8, 1024] (server hard constraint, undocumented).
|
// Restore the batch-size clamping warning that was lost when the logic moved
|
||||||
// Surface the clamp on stderr instead of silently rewriting the user's
|
// into profiles. The profile silently clamps to [8, 1024]; surface it here
|
||||||
// value — otherwise the submitted body would carry a number the user never
|
// so the user has an audit trail. Skip modalities that bypass the batch_size
|
||||||
// typed, with no audit trail. (Range observed on common SFT / SFT-LoRA
|
// gate (image/video): their batch_size is a fixed model-family default, not
|
||||||
// training types; some bases like qwen3.6-flash report a wider range, so
|
// a clamp of the user's value, so the [8, 1024] "clamped" message would be
|
||||||
// the warning explicitly mentions "server range".)
|
// self-contradictory (video uses 2/4) and misleading.
|
||||||
if (flags.batchSize !== undefined) {
|
if (
|
||||||
|
flags.batchSize !== undefined &&
|
||||||
|
hp.batch_size !== undefined &&
|
||||||
|
!settings.quiet &&
|
||||||
|
!profile.shouldSkipGate("batch_size", modality)
|
||||||
|
) {
|
||||||
const requested = flags.batchSize;
|
const requested = flags.batchSize;
|
||||||
let batchSize = requested;
|
if (hp.batch_size !== requested) {
|
||||||
if (batchSize < 8) batchSize = 8;
|
|
||||||
if (batchSize > 1024) batchSize = 1024;
|
|
||||||
if (batchSize !== requested && !settings.quiet) {
|
|
||||||
process.stderr.write(
|
process.stderr.write(
|
||||||
`warning: --batch-size ${requested} clamped to ${batchSize} ` +
|
`warning: --batch-size ${requested} clamped to ${hp.batch_size} ` +
|
||||||
`(server range [8, 1024] for the common training types).\n`,
|
`(server range [8, 1024] for the common training types).\n`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
hp.batch_size = batchSize;
|
}
|
||||||
|
// For modalities that skip the batch_size gate (video: fixed 2/4 by model
|
||||||
|
// family), warn the user that their explicit --batch-size was discarded.
|
||||||
|
if (
|
||||||
|
flags.batchSize !== undefined &&
|
||||||
|
!settings.quiet &&
|
||||||
|
profile.shouldSkipGate("batch_size", modality)
|
||||||
|
) {
|
||||||
|
const requested = flags.batchSize;
|
||||||
|
if (hp.batch_size !== undefined && hp.batch_size !== requested) {
|
||||||
|
process.stderr.write(
|
||||||
|
`warning: --batch-size ${requested} ignored for ${modality} training ` +
|
||||||
|
`(model uses a fixed batch_size of ${hp.batch_size}).\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto batch_size for small datasets: fetch first training file size.
|
// Auto batch_size for small datasets — only for text data. Audio/image/video
|
||||||
// With default split=0.9, validation_set = 0.1 * rows.
|
// profiles already set their own batch parameters.
|
||||||
// Platform default batch_size=16 needs rows > 160; batch_size=8 needs rows > 80.
|
if (modality === "text" && hp.batch_size === undefined && !settings.dryRun) {
|
||||||
// Files < 100KB are conservatively estimated to have < 200 rows.
|
|
||||||
// If the first file was just uploaded we already hold its size; otherwise
|
|
||||||
// fall back to getDataset.
|
|
||||||
if (hp.batch_size === undefined && !settings.dryRun) {
|
|
||||||
let sizeBytes = training.firstSize ?? 0;
|
let sizeBytes = training.firstSize ?? 0;
|
||||||
if (sizeBytes === 0) {
|
if (sizeBytes === 0) {
|
||||||
try {
|
try {
|
||||||
@@ -396,7 +438,11 @@ export default defineCommand({
|
|||||||
// code as `validateDataset`) so the failure surfaces through the same
|
// code as `validateDataset`) so the failure surfaces through the same
|
||||||
// `BailianError` + issue convention used by `dataset upload`/`validate`.
|
// `BailianError` + issue convention used by `dataset upload`/`validate`.
|
||||||
// ExitCode.GENERAL matches the existing validation-failed exit code.
|
// ExitCode.GENERAL matches the existing validation-failed exit code.
|
||||||
if (!settings.dryRun && training.recordCount !== undefined) {
|
if (
|
||||||
|
!settings.dryRun &&
|
||||||
|
training.recordCount !== undefined &&
|
||||||
|
!profile.shouldSkipGate("batch_size", modality)
|
||||||
|
) {
|
||||||
// 16 is the platform default when neither the user nor the small-file
|
// 16 is the platform default when neither the user nor the small-file
|
||||||
// auto-adjust set a batch_size (see the auto-adjust comment above).
|
// auto-adjust set a batch_size (see the auto-adjust comment above).
|
||||||
const effectiveBatchSize = hp.batch_size ?? 16;
|
const effectiveBatchSize = hp.batch_size ?? 16;
|
||||||
@@ -415,7 +461,7 @@ export default defineCommand({
|
|||||||
// be trained against. listFoundationModels is a public API (no console
|
// be trained against. listFoundationModels is a public API (no console
|
||||||
// login required); on lookup failure (network / 401 / etc.) we fall back
|
// login required); on lookup failure (network / 401 / etc.) we fall back
|
||||||
// to letting the server decide rather than blocking the submit.
|
// to letting the server decide rather than blocking the submit.
|
||||||
if (!settings.dryRun) {
|
if (!settings.dryRun && !profile.shouldSkipCapabilityCheck(modality)) {
|
||||||
let capability: Awaited<ReturnType<typeof fetchModelCapability>> | undefined;
|
let capability: Awaited<ReturnType<typeof fetchModelCapability>> | undefined;
|
||||||
try {
|
try {
|
||||||
capability = await fetchModelCapability(settings, model);
|
capability = await fetchModelCapability(settings, model);
|
||||||
@@ -453,8 +499,8 @@ export default defineCommand({
|
|||||||
const body: CreateFineTuneRequest = {
|
const body: CreateFineTuneRequest = {
|
||||||
model,
|
model,
|
||||||
training_file_ids: trainingFileIds,
|
training_file_ids: trainingFileIds,
|
||||||
// Map the CLI training type to the server value at the interface boundary.
|
// Profile maps the CLI training type to the server value at the boundary.
|
||||||
training_type: toServerTrainingType(trainingType),
|
training_type: profile.serverTrainingType,
|
||||||
hyper_parameters: hp,
|
hyper_parameters: hp,
|
||||||
};
|
};
|
||||||
if (validationFileIds && validationFileIds.length > 0) {
|
if (validationFileIds && validationFileIds.length > 0) {
|
||||||
|
|||||||
@@ -40,10 +40,12 @@
|
|||||||
"check": "vp check"
|
"check": "vp check"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"yaml": "^2.8.3"
|
"yaml": "^2.8.3",
|
||||||
|
"yauzl": "catalog:"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "catalog:",
|
"@types/node": "catalog:",
|
||||||
|
"@types/yauzl": "catalog:",
|
||||||
"@typescript/native-preview": "7.0.0-dev.20260328.1",
|
"@typescript/native-preview": "7.0.0-dev.20260328.1",
|
||||||
"typescript": "^6.0.2",
|
"typescript": "^6.0.2",
|
||||||
"vite-plus": "catalog:"
|
"vite-plus": "catalog:"
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
export * from "./types.ts";
|
export * from "./types.ts";
|
||||||
export * from "./api.ts";
|
export * from "./api.ts";
|
||||||
|
export { detectModality } from "./inspect.ts";
|
||||||
export {
|
export {
|
||||||
validateDataset,
|
validateDataset,
|
||||||
pickValidator,
|
pickValidator,
|
||||||
registerValidator,
|
registerValidator,
|
||||||
listSupportedFormats,
|
listSupportedFormats,
|
||||||
MAX_DATASET_BYTES,
|
MAX_DATASET_BYTES,
|
||||||
|
MAX_MEDIA_ZIP_BYTES,
|
||||||
parseDatasetSchemaFlag,
|
parseDatasetSchemaFlag,
|
||||||
formatIssue,
|
formatIssue,
|
||||||
} from "./validate/index.ts";
|
} from "./validate/index.ts";
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
/**
|
||||||
|
* Data inspector — lightweight content parser that determines the modality
|
||||||
|
* of a training data file.
|
||||||
|
*
|
||||||
|
* The inspector peeks at the first non-blank line of a JSONL file (or the
|
||||||
|
* `data.jsonl` manifest inside a ZIP) and inspects the record's fields to
|
||||||
|
* decide whether the data carries text, audio, image, or video samples.
|
||||||
|
*
|
||||||
|
* This is intentionally shallow — it reads at most one record — so it stays
|
||||||
|
* fast even on very large files. The full structural validation is the job
|
||||||
|
* of the format-specific validator (`jsonl.ts`, `zip.ts`), not this module.
|
||||||
|
*
|
||||||
|
* Routing in `create.ts`:
|
||||||
|
* 1. `--training-type` → Profile (via `getProfile`)
|
||||||
|
* 2. Profile.acceptedExtensions → match file extension
|
||||||
|
* 3. `detectModality(filePath)` → "text" | "audio" | "image" | "video"
|
||||||
|
* 4. Profile validates / resolves hyper-params using detected modality
|
||||||
|
*/
|
||||||
|
import { createReadStream } from "fs";
|
||||||
|
import { createInterface } from "readline";
|
||||||
|
import { extname } from "path";
|
||||||
|
import { BailianError } from "../errors/base.ts";
|
||||||
|
import { ExitCode } from "../errors/codes.ts";
|
||||||
|
import type { DataModality } from "../finetune/profiles/types.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inspect a file and return its data modality.
|
||||||
|
*
|
||||||
|
* `.jsonl` → read the first non-blank line, parse JSON, check fields.
|
||||||
|
* `.zip` → locate `data.jsonl` inside the archive, read its first line.
|
||||||
|
*
|
||||||
|
* Image data returns `"image"` (T2I) or `"image-i2i"` (I2I, first record
|
||||||
|
* has `input_img`). Callers that don't distinguish can normalise to `"image"`.
|
||||||
|
*
|
||||||
|
* Throws USAGE if the file extension is not `.jsonl` or `.zip`, or if the
|
||||||
|
* content cannot be parsed.
|
||||||
|
*/
|
||||||
|
export async function detectModality(filePath: string): Promise<DataModality> {
|
||||||
|
const ext = extname(filePath).toLowerCase();
|
||||||
|
if (ext === ".jsonl") return detectFromJsonl(filePath);
|
||||||
|
if (ext === ".zip") return detectFromZip(filePath);
|
||||||
|
throw new BailianError(
|
||||||
|
`Cannot inspect file with extension "${ext}". Expected .jsonl or .zip.`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the first non-blank line of a JSONL file and determine the modality.
|
||||||
|
*/
|
||||||
|
async function detectFromJsonl(filePath: string): Promise<DataModality> {
|
||||||
|
const firstLine = await readFirstNonBlankLine(filePath);
|
||||||
|
if (!firstLine) {
|
||||||
|
throw new BailianError(
|
||||||
|
`JSONL file is empty or contains only blank lines: ${filePath}`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const modality = classifyRecord(firstLine);
|
||||||
|
// JSONL files are always text data (chatml / dpo / cpt).
|
||||||
|
return modality === "unknown" ? "text" : modality;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Locate `data.jsonl` inside a ZIP archive, extract its first non-blank line,
|
||||||
|
* and determine the modality.
|
||||||
|
*
|
||||||
|
* Uses `yauzl` for streaming access — only the target entry is read, the rest
|
||||||
|
* of the archive is skipped.
|
||||||
|
*/
|
||||||
|
async function detectFromZip(filePath: string): Promise<DataModality> {
|
||||||
|
const firstLine = await readFirstLineFromZipEntry(filePath, "data.jsonl");
|
||||||
|
if (!firstLine) {
|
||||||
|
throw new BailianError(
|
||||||
|
`ZIP archive does not contain "data.jsonl" or it is empty: ${filePath}`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
`Audio training data must be a ZIP with data.jsonl at the root and a train/ subfolder.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const modality = classifyRecord(firstLine);
|
||||||
|
if (modality === "unknown") {
|
||||||
|
throw new BailianError(
|
||||||
|
`ZIP data.jsonl does not match any supported media format ` +
|
||||||
|
`(expected wav_fn / img_path / first_frame_path / video_path): ${filePath}`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
`ZIP archives are for audio/image/video training data. ` +
|
||||||
|
`For text data, use a .jsonl file instead.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return modality;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Classify a JSON record into a data modality based on its field names. */
|
||||||
|
function classifyRecord(line: string): DataModality | "unknown" {
|
||||||
|
let record: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
record = JSON.parse(line);
|
||||||
|
} catch {
|
||||||
|
throw new BailianError(
|
||||||
|
`Failed to parse first JSON record for modality detection: ${line.slice(0, 120)}`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (typeof record !== "object" || record === null || Array.isArray(record)) {
|
||||||
|
throw new BailianError(
|
||||||
|
`Expected a JSON object as the first record, got ${Array.isArray(record) ? "array" : typeof record}.`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if ("wav_fn" in record) return "audio";
|
||||||
|
if ("img_path" in record) {
|
||||||
|
// Image generation: distinguish T2I (no input_img) from I2I (has input_img).
|
||||||
|
// The subtype drives hyper-parameter defaults (max_pixels 2k vs 1k).
|
||||||
|
return "input_img" in record ? "image-i2i" : "image";
|
||||||
|
}
|
||||||
|
if ("first_frame_path" in record || "video_path" in record) {
|
||||||
|
// Video generation (Wan i2v/kf2v): distinguish first-frame-only (i2v) from
|
||||||
|
// first+last-frame (kf2v, has last_frame_path). The subtype lets the
|
||||||
|
// profile cross-check the chosen --model against the data shape.
|
||||||
|
return "last_frame_path" in record ? "video-kf2v" : "video";
|
||||||
|
}
|
||||||
|
// No known media field found — caller decides how to handle.
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read the first non-blank line from a file using a readline stream. */
|
||||||
|
function readFirstNonBlankLine(filePath: string): Promise<string | null> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||||
|
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||||
|
let found = false;
|
||||||
|
rl.on("line", (line) => {
|
||||||
|
if (found) return;
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (trimmed.length === 0) return;
|
||||||
|
found = true;
|
||||||
|
rl.close();
|
||||||
|
stream.destroy();
|
||||||
|
resolve(trimmed);
|
||||||
|
});
|
||||||
|
rl.on("close", () => {
|
||||||
|
if (!found) resolve(null);
|
||||||
|
});
|
||||||
|
rl.on("error", reject);
|
||||||
|
stream.on("error", reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open a ZIP archive, locate the entry with the given name, and return the
|
||||||
|
* first non-blank line from its content. Returns `null` if the entry is not
|
||||||
|
* found or is empty.
|
||||||
|
*
|
||||||
|
* Delegates ZIP open/locate to the shared `openZipAndFindEntry` helper in
|
||||||
|
* `validate/zip.ts` — avoids duplicating yauzl boilerplate.
|
||||||
|
*/
|
||||||
|
function readFirstLineFromZipEntry(zipPath: string, entryName: string): Promise<string | null> {
|
||||||
|
return import("./validate/zip.ts")
|
||||||
|
.then(({ openZipAndFindEntry }) => openZipAndFindEntry(zipPath, entryName))
|
||||||
|
.then(({ entry, zipfile }) => {
|
||||||
|
return new Promise<string | null>((resolve, reject) => {
|
||||||
|
zipfile.openReadStream(entry, (streamErr, readStream) => {
|
||||||
|
if (streamErr || !readStream) {
|
||||||
|
zipfile.close();
|
||||||
|
reject(
|
||||||
|
new BailianError(
|
||||||
|
`Failed to read "${entryName}" from ZIP: ${streamErr?.message}`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const rl = createInterface({ input: readStream, crlfDelay: Infinity });
|
||||||
|
let found = false;
|
||||||
|
rl.on("line", (line) => {
|
||||||
|
if (found) return;
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (trimmed.length === 0) return;
|
||||||
|
found = true;
|
||||||
|
rl.close();
|
||||||
|
readStream.destroy();
|
||||||
|
zipfile.close();
|
||||||
|
resolve(trimmed);
|
||||||
|
});
|
||||||
|
rl.on("close", () => {
|
||||||
|
if (!found) {
|
||||||
|
zipfile.close();
|
||||||
|
resolve(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
rl.on("error", (readError) => {
|
||||||
|
zipfile.close();
|
||||||
|
reject(readError);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
// openZipAndFindEntry rejects when the entry is not found — treat as null.
|
||||||
|
if (error instanceof Error && error.message.includes("not found in ZIP")) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -18,6 +18,13 @@ import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts
|
|||||||
*/
|
*/
|
||||||
export const MAX_DATASET_BYTES = 300 * 1024 * 1024;
|
export const MAX_DATASET_BYTES = 300 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Image / video ZIP size cap — 1 GB per the platform docs (vs 300 MB for
|
||||||
|
* text / audio). Used by `bl dataset upload` for media schemas and by the
|
||||||
|
* `sft-lora` training profile for image / video validation.
|
||||||
|
*/
|
||||||
|
export const MAX_MEDIA_ZIP_BYTES = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
export interface PreflightResult {
|
export interface PreflightResult {
|
||||||
bytes: number;
|
bytes: number;
|
||||||
ext: string;
|
ext: string;
|
||||||
@@ -75,11 +82,12 @@ export function emptyStats(): ValidationStats {
|
|||||||
export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined {
|
export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined {
|
||||||
if (value === undefined || value.trim() === "") return undefined;
|
if (value === undefined || value.trim() === "") return undefined;
|
||||||
const v = value.trim();
|
const v = value.trim();
|
||||||
if (v === "chatml" || v === "dpo" || v === "cpt") return v;
|
if (v === "chatml" || v === "dpo" || v === "cpt" || v === "tts" || v === "image" || v === "video")
|
||||||
|
return v;
|
||||||
throw new BailianError(
|
throw new BailianError(
|
||||||
`Unsupported --schema "${value}". Supported: chatml, dpo, cpt.`,
|
`Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image, video.`,
|
||||||
ExitCode.USAGE,
|
ExitCode.USAGE,
|
||||||
`Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, else ChatML).`,
|
`Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, first_frame_path/video_path → video, else ChatML).`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ export {
|
|||||||
registerValidator,
|
registerValidator,
|
||||||
listSupportedFormats,
|
listSupportedFormats,
|
||||||
} from "./registry.ts";
|
} from "./registry.ts";
|
||||||
export { MAX_DATASET_BYTES, parseDatasetSchemaFlag } from "./common.ts";
|
export { MAX_DATASET_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag } from "./common.ts";
|
||||||
export { formatIssue } from "./format.ts";
|
export { formatIssue } from "./format.ts";
|
||||||
export type {
|
export type {
|
||||||
ValidatorSpec,
|
ValidatorSpec,
|
||||||
|
|||||||
@@ -16,10 +16,11 @@ import { extname } from "path";
|
|||||||
import { BailianError } from "../../errors/base.ts";
|
import { BailianError } from "../../errors/base.ts";
|
||||||
import { ExitCode } from "../../errors/codes.ts";
|
import { ExitCode } from "../../errors/codes.ts";
|
||||||
import { jsonlValidator } from "./jsonl.ts";
|
import { jsonlValidator } from "./jsonl.ts";
|
||||||
|
import { zipValidator } from "./zip.ts";
|
||||||
import { preflight, MAX_DATASET_BYTES } from "./common.ts";
|
import { preflight, MAX_DATASET_BYTES } from "./common.ts";
|
||||||
import type { ValidatorSpec, ValidateOpts, ValidationResult } from "./types.ts";
|
import type { ValidatorSpec, ValidateOpts, ValidationResult } from "./types.ts";
|
||||||
|
|
||||||
const REGISTRY: ValidatorSpec[] = [jsonlValidator];
|
const REGISTRY: ValidatorSpec[] = [jsonlValidator, zipValidator];
|
||||||
|
|
||||||
/** Lookup the validator that handles a given file extension. */
|
/** Lookup the validator that handles a given file extension. */
|
||||||
export function pickValidator(filePath: string): ValidatorSpec {
|
export function pickValidator(filePath: string): ValidatorSpec {
|
||||||
|
|||||||
@@ -0,0 +1,177 @@
|
|||||||
|
/**
|
||||||
|
* Image generation record schema — Wan2.x fine-tuning.
|
||||||
|
*
|
||||||
|
* Two record flavours share the same schema:
|
||||||
|
* - **Text-to-image (T2I):** `{"prompt": "...", "img_path": "./x.png"}`
|
||||||
|
* - **Image-to-image (I2I):** `{"prompt": "...", "input_img": "./in.jpg", "img_path": "./out.jpg"}`
|
||||||
|
*
|
||||||
|
* The presence of `img_path` is the distinguishing field — auto-detect picks
|
||||||
|
* this schema before the ChatML fallback. `input_img` is optional (I2I only).
|
||||||
|
*
|
||||||
|
* Image data lives in a ZIP with a flat layout (no `train/` subdirectory).
|
||||||
|
* File names must be ASCII-only per platform requirements.
|
||||||
|
*/
|
||||||
|
import { makeIssue } from "../common.ts";
|
||||||
|
import type { ValidationIssue } from "../types.ts";
|
||||||
|
import type { RecordSchemaSpec } from "./types.ts";
|
||||||
|
|
||||||
|
/** Accepted image file extensions (lower-case, with dot). */
|
||||||
|
export const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tiff"]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check that a path string ends with an accepted image extension.
|
||||||
|
* Returns the extension (lower-case) or an empty string.
|
||||||
|
*/
|
||||||
|
function imageExt(path: string): string {
|
||||||
|
const dot = path.lastIndexOf(".");
|
||||||
|
return dot >= 0 ? path.slice(dot).toLowerCase() : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Warn (non-error) when a filename contains non-ASCII characters.
|
||||||
|
* The platform requires English-only filenames.
|
||||||
|
*/
|
||||||
|
function asciiOnly(value: string): boolean {
|
||||||
|
return /^[\x20-\x7E]+$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inspectImageRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
|
||||||
|
const out: ValidationIssue[] = [];
|
||||||
|
|
||||||
|
// --- prompt (required) ---
|
||||||
|
if (!("prompt" in record)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_PROMPT", `Required field "prompt" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const prompt = record.prompt;
|
||||||
|
if (typeof prompt !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "INVALID_PROMPT", `"prompt" must be a string (got ${typeof prompt}).`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else if (prompt.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_PROMPT", `"prompt" must not be empty / whitespace-only.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- img_path (required) ---
|
||||||
|
if (!("img_path" in record)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_IMG_PATH", `Required field "img_path" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "img_path",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const imgPath = record.img_path;
|
||||||
|
if (typeof imgPath !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"INVALID_IMG_PATH",
|
||||||
|
`"img_path" must be a string (got ${typeof imgPath}).`,
|
||||||
|
{ line: lineNo, path: "img_path" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else if (imgPath.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_IMG_PATH", `"img_path" must not be empty.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "img_path",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const ext = imageExt(imgPath);
|
||||||
|
if (!IMAGE_EXTENSIONS.has(ext)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"UNUSUAL_IMAGE_EXT",
|
||||||
|
`"img_path" points to a non-standard image extension "${ext || "(none)"}". ` +
|
||||||
|
`Expected one of: ${[...IMAGE_EXTENSIONS].join(", ")}.`,
|
||||||
|
{ line: lineNo, path: "img_path" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!asciiOnly(imgPath)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"NON_ASCII_IMG_PATH",
|
||||||
|
`"img_path" must contain only ASCII characters (English filenames required). Got: "${imgPath}".`,
|
||||||
|
{ line: lineNo, path: "img_path" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- input_img (optional — present only for I2I records) ---
|
||||||
|
if ("input_img" in record) {
|
||||||
|
const inputImg = record.input_img;
|
||||||
|
if (typeof inputImg !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"INVALID_INPUT_IMG",
|
||||||
|
`"input_img" must be a string (got ${typeof inputImg}).`,
|
||||||
|
{ line: lineNo, path: "input_img" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else if (inputImg.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_INPUT_IMG", `"input_img" must not be empty.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "input_img",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const ext = imageExt(inputImg);
|
||||||
|
if (!IMAGE_EXTENSIONS.has(ext)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"UNUSUAL_INPUT_IMG_EXT",
|
||||||
|
`"input_img" points to a non-standard image extension "${ext || "(none)"}". ` +
|
||||||
|
`Expected one of: ${[...IMAGE_EXTENSIONS].join(", ")}.`,
|
||||||
|
{ line: lineNo, path: "input_img" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!asciiOnly(inputImg)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"NON_ASCII_INPUT_IMG",
|
||||||
|
`"input_img" must contain only ASCII characters (English filenames required). Got: "${inputImg}".`,
|
||||||
|
{ line: lineNo, path: "input_img" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Image generation schema. Auto-detect: a record matches when it carries
|
||||||
|
* `img_path`. Placed before ChatML in the registry so image data is never
|
||||||
|
* misclassified.
|
||||||
|
*/
|
||||||
|
export const imageSchema: RecordSchemaSpec = {
|
||||||
|
name: "image",
|
||||||
|
detect: (record) => "img_path" in record,
|
||||||
|
inspect: inspectImageRecord,
|
||||||
|
};
|
||||||
@@ -16,11 +16,21 @@ import type { RecordSchemaSpec } from "./types.ts";
|
|||||||
import { chatmlSchema } from "./chatml.ts";
|
import { chatmlSchema } from "./chatml.ts";
|
||||||
import { cptSchema } from "./cpt.ts";
|
import { cptSchema } from "./cpt.ts";
|
||||||
import { dpoSchema } from "./dpo.ts";
|
import { dpoSchema } from "./dpo.ts";
|
||||||
|
import { ttsSchema } from "./tts.ts";
|
||||||
|
import { imageSchema } from "./image.ts";
|
||||||
|
import { videoSchema } from "./video.ts";
|
||||||
|
|
||||||
// Order matters: DPO (chosen/rejected) and CPT (text) before ChatML (the
|
// Order matters: TTS (wav_fn), image (img_path), video (first_frame_path/
|
||||||
// catch-all fallback). Each keys off a distinguishing field so the three
|
// video_path), DPO (chosen/rejected) and CPT (text) before ChatML (the catch-
|
||||||
// partition cleanly — DPO never looks like CPT, etc.
|
// all fallback). Each keys off a distinguishing field so they partition cleanly.
|
||||||
export const RECORD_SCHEMAS: RecordSchemaSpec[] = [dpoSchema, cptSchema, chatmlSchema];
|
export const RECORD_SCHEMAS: RecordSchemaSpec[] = [
|
||||||
|
ttsSchema,
|
||||||
|
imageSchema,
|
||||||
|
videoSchema,
|
||||||
|
dpoSchema,
|
||||||
|
cptSchema,
|
||||||
|
chatmlSchema,
|
||||||
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Pick the right schema for a single parsed record.
|
* Pick the right schema for a single parsed record.
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
/**
|
||||||
|
* TTS record schema — `{"wav_fn": "train/xxx.wav", "text": "..."}`.
|
||||||
|
*
|
||||||
|
* Used for audio fine-tuning (e.g. CosyVoice v3 Flash). Each JSONL record
|
||||||
|
* inside the training data ZIP's `data.jsonl` maps a `.wav` file path to its
|
||||||
|
* transcript. The ZIP validator (`../zip.ts`) calls into this schema via the
|
||||||
|
* standard `jsonlValidator` pipeline — the schema only owns per-record checks,
|
||||||
|
* the ZIP-level structural validation is separate.
|
||||||
|
*
|
||||||
|
* Auto-detect: a record matches when it carries `wav_fn` — this is unique to
|
||||||
|
* audio training data and will never collide with ChatML/DPO/CPT.
|
||||||
|
*/
|
||||||
|
import { makeIssue } from "../common.ts";
|
||||||
|
import type { ValidationIssue } from "../types.ts";
|
||||||
|
import type { RecordSchemaSpec } from "./types.ts";
|
||||||
|
|
||||||
|
/** Expected audio file extensions (lower-case, with dot). */
|
||||||
|
const AUDIO_EXTENSIONS = new Set([".wav", ".mp3", ".flac", ".ogg", ".m4a"]);
|
||||||
|
|
||||||
|
function inspectTTSRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
|
||||||
|
const out: ValidationIssue[] = [];
|
||||||
|
|
||||||
|
// --- wav_fn ---
|
||||||
|
if (!("wav_fn" in record)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_WAV_FN", `Required field "wav_fn" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "wav_fn",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const wavFn = record.wav_fn;
|
||||||
|
if (typeof wavFn !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "INVALID_WAV_FN", `"wav_fn" must be a string (got ${typeof wavFn}).`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "wav_fn",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else if (wavFn.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_WAV_FN", `"wav_fn" must not be empty.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "wav_fn",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// Check that the path looks like it references an audio file.
|
||||||
|
const dotIndex = wavFn.lastIndexOf(".");
|
||||||
|
const ext = dotIndex >= 0 ? wavFn.slice(dotIndex).toLowerCase() : "";
|
||||||
|
if (!AUDIO_EXTENSIONS.has(ext)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"UNUSUAL_AUDIO_EXT",
|
||||||
|
`"wav_fn" points to a non-standard audio extension "${ext || "(none)"}". ` +
|
||||||
|
`Expected one of: ${[...AUDIO_EXTENSIONS].join(", ")}.`,
|
||||||
|
{ line: lineNo, path: "wav_fn" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- text ---
|
||||||
|
if (!("text" in record)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_TEXT", `Required field "text" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "text",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const text = record.text;
|
||||||
|
if (typeof text !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "INVALID_TEXT", `"text" must be a string (got ${typeof text}).`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "text",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else if (text.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_TEXT", `"text" must not be empty / whitespace-only.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "text",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TTS schema. Auto-detect: a record is treated as TTS when it carries `wav_fn`.
|
||||||
|
* Placed first in the registry so audio data is never misclassified as ChatML.
|
||||||
|
*/
|
||||||
|
export const ttsSchema: RecordSchemaSpec = {
|
||||||
|
name: "tts",
|
||||||
|
detect: (record) => "wav_fn" in record,
|
||||||
|
inspect: inspectTTSRecord,
|
||||||
|
};
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
/**
|
||||||
|
* Video generation record schema — Wan i2v / kf2v fine-tuning.
|
||||||
|
*
|
||||||
|
* Two record flavours share the same schema:
|
||||||
|
* - **Image-to-video, first frame (i2v):**
|
||||||
|
* `{"prompt": "...", "first_frame_path": "image_1.jpg", "video_path": "video_1.mp4"}`
|
||||||
|
* - **Image-to-video, first+last frame (kf2v):**
|
||||||
|
* `{"prompt": "...", "first_frame_path": "image/x_first.jpg",
|
||||||
|
* "last_frame_path": "image/x_last.jpg", "video_path": "video/x.mp4"}`
|
||||||
|
*
|
||||||
|
* The presence of `first_frame_path` / `video_path` is the distinguishing
|
||||||
|
* signal — auto-detect picks this schema before the ChatML fallback.
|
||||||
|
*
|
||||||
|
* `video_path` is OPTIONAL: validation-set records omit the target video (the
|
||||||
|
* platform generates preview videos from the first frame + prompt at each eval
|
||||||
|
* checkpoint), so the same schema validates both training and validation zips.
|
||||||
|
* `last_frame_path` is optional (kf2v only).
|
||||||
|
*
|
||||||
|
* Video data lives in a ZIP: i2v is flat, kf2v uses `image/` + `video/`
|
||||||
|
* subfolders. File names should be ASCII-only per platform requirements.
|
||||||
|
*/
|
||||||
|
import { makeIssue } from "../common.ts";
|
||||||
|
import type { ValidationIssue } from "../types.ts";
|
||||||
|
import type { RecordSchemaSpec } from "./types.ts";
|
||||||
|
|
||||||
|
/** Accepted image (frame) file extensions (lower-case, with dot). */
|
||||||
|
export const VIDEO_IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp"]);
|
||||||
|
|
||||||
|
/** Accepted video file extensions (lower-case, with dot). */
|
||||||
|
export const VIDEO_EXTENSIONS = new Set([".mp4", ".mov"]);
|
||||||
|
|
||||||
|
/** Return the lower-case extension of a path (with dot), or "". */
|
||||||
|
function pathExt(path: string): string {
|
||||||
|
const dot = path.lastIndexOf(".");
|
||||||
|
return dot >= 0 ? path.slice(dot).toLowerCase() : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The platform requires English-only (ASCII) file names. */
|
||||||
|
function asciiOnly(value: string): boolean {
|
||||||
|
return /^[\x20-\x7E]+$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate a required string path field, checking its extension against the
|
||||||
|
* accepted set and warning on non-ASCII names. Pushes issues into `out`.
|
||||||
|
*/
|
||||||
|
function checkPathField(
|
||||||
|
out: ValidationIssue[],
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
field: string,
|
||||||
|
required: boolean,
|
||||||
|
accepted: Set<string>,
|
||||||
|
lineNo: number,
|
||||||
|
): void {
|
||||||
|
if (!(field in record)) {
|
||||||
|
if (required) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_FIELD", `Required field "${field}" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: field,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const value = record[field];
|
||||||
|
if (typeof value !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "INVALID_FIELD", `"${field}" must be a string (got ${typeof value}).`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: field,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (value.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_FIELD", `"${field}" must not be empty.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: field,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const ext = pathExt(value);
|
||||||
|
if (!accepted.has(ext)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"UNUSUAL_MEDIA_EXT",
|
||||||
|
`"${field}" points to a non-standard extension "${ext || "(none)"}". ` +
|
||||||
|
`Expected one of: ${[...accepted].join(", ")}.`,
|
||||||
|
{ line: lineNo, path: field },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!asciiOnly(value)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"NON_ASCII_PATH",
|
||||||
|
`"${field}" must contain only ASCII characters (English filenames required). Got: "${value}".`,
|
||||||
|
{ line: lineNo, path: field },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function inspectVideoRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
|
||||||
|
const out: ValidationIssue[] = [];
|
||||||
|
|
||||||
|
// --- prompt (required) ---
|
||||||
|
if (!("prompt" in record)) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "MISSING_PROMPT", `Required field "prompt" is missing.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const prompt = record.prompt;
|
||||||
|
if (typeof prompt !== "string") {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "INVALID_PROMPT", `"prompt" must be a string (got ${typeof prompt}).`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else if (prompt.trim().length === 0) {
|
||||||
|
out.push(
|
||||||
|
makeIssue("error", "EMPTY_PROMPT", `"prompt" must not be empty / whitespace-only.`, {
|
||||||
|
line: lineNo,
|
||||||
|
path: "prompt",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- first_frame_path (required) ---
|
||||||
|
checkPathField(out, record, "first_frame_path", true, VIDEO_IMAGE_EXTENSIONS, lineNo);
|
||||||
|
// --- last_frame_path (optional — kf2v only) ---
|
||||||
|
checkPathField(out, record, "last_frame_path", false, VIDEO_IMAGE_EXTENSIONS, lineNo);
|
||||||
|
// --- video_path (optional — training only; validation sets omit it) ---
|
||||||
|
checkPathField(out, record, "video_path", false, VIDEO_EXTENSIONS, lineNo);
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Video generation schema. Auto-detect: a record matches when it carries
|
||||||
|
* `first_frame_path` or `video_path`. Placed before ChatML in the registry so
|
||||||
|
* video data is never misclassified. Distinct from image (`img_path`).
|
||||||
|
*/
|
||||||
|
export const videoSchema: RecordSchemaSpec = {
|
||||||
|
name: "video",
|
||||||
|
detect: (record) => "first_frame_path" in record || "video_path" in record,
|
||||||
|
inspect: inspectVideoRecord,
|
||||||
|
};
|
||||||
@@ -32,10 +32,17 @@ export interface ValidateOpts {
|
|||||||
* platform ten minutes in.
|
* platform ten minutes in.
|
||||||
*/
|
*/
|
||||||
schema?: DatasetSchema;
|
schema?: DatasetSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Model identifier (`--model`) forwarded for schema-agnostic cross-checks —
|
||||||
|
* e.g. the video validator uses it to verify a `kf2v` model is paired with
|
||||||
|
* first+last-frame data. Optional: absent for bare file-id flows.
|
||||||
|
*/
|
||||||
|
model?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The schemas a `.jsonl` record can be validated against. */
|
/** The schemas a `.jsonl` record can be validated against. */
|
||||||
export type DatasetSchema = "chatml" | "dpo" | "cpt";
|
export type DatasetSchema = "chatml" | "dpo" | "cpt" | "tts" | "image" | "video";
|
||||||
|
|
||||||
export type ValidationSeverity = "error" | "warning";
|
export type ValidationSeverity = "error" | "warning";
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,371 @@
|
|||||||
|
/**
|
||||||
|
* ZIP validator — audio / image / video training data archives.
|
||||||
|
*
|
||||||
|
* A training data ZIP must have:
|
||||||
|
* - `data.jsonl` at the root — the manifest mapping media files to labels.
|
||||||
|
* - A `train/` subfolder (or media files at the root) referenced by the
|
||||||
|
* manifest entries.
|
||||||
|
*
|
||||||
|
* This validator owns the **ZIP-level structural checks** (entries present,
|
||||||
|
* references resolve). The **per-record JSONL content validation** is delegated
|
||||||
|
* to the existing `jsonlValidator` — we extract `data.jsonl` to a temp file,
|
||||||
|
* run the full pipeline (quickScan + deepCheck + schema dispatch), and stitch
|
||||||
|
* the results together.
|
||||||
|
*
|
||||||
|
* The schema for `data.jsonl` records is passed via `opts.schema` (typically
|
||||||
|
* `"tts"` for audio). The profile layer decides which schema to use based on
|
||||||
|
* the detected modality — this validator is schema-agnostic.
|
||||||
|
*/
|
||||||
|
import { createWriteStream, mkdirSync, rmSync } from "fs";
|
||||||
|
import { tmpdir } from "os";
|
||||||
|
import { join } from "path";
|
||||||
|
import { pipeline } from "stream/promises";
|
||||||
|
import { randomBytes } from "crypto";
|
||||||
|
import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts";
|
||||||
|
import { makeIssue } from "./common.ts";
|
||||||
|
import { jsonlValidator } from "./jsonl.ts";
|
||||||
|
import { IMAGE_EXTENSIONS } from "./schemas/image.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open a ZIP archive and locate a specific entry by name.
|
||||||
|
* Returns the entry and zipfile handle — **caller must close the zipfile**.
|
||||||
|
* Normalises entry names (backslash → forward-slash) and supports entries
|
||||||
|
* at the root or inside subdirectories (matches `name === targetName` or
|
||||||
|
* `name.endsWith("/${targetName}")`).
|
||||||
|
*
|
||||||
|
* Shared by `extractZipEntry` (this file) and `readFirstLineFromZipEntry`
|
||||||
|
* (inspect.ts) to avoid duplicating yauzl open/iterate/locate boilerplate.
|
||||||
|
*/
|
||||||
|
export function openZipAndFindEntry(
|
||||||
|
zipPath: string,
|
||||||
|
targetName: string,
|
||||||
|
): Promise<{ entry: import("yauzl").Entry; zipfile: import("yauzl").ZipFile }> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
import("yauzl")
|
||||||
|
.then((yauzl) => {
|
||||||
|
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
|
||||||
|
if (err || !zipfile) {
|
||||||
|
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
zipfile.readEntry();
|
||||||
|
zipfile.on("entry", (entry) => {
|
||||||
|
const name = entry.fileName.replace(/\\/g, "/");
|
||||||
|
if (name === targetName || name.endsWith(`/${targetName}`)) {
|
||||||
|
resolve({ entry, zipfile });
|
||||||
|
} else {
|
||||||
|
zipfile.readEntry();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
zipfile.on("end", () => {
|
||||||
|
zipfile.close();
|
||||||
|
reject(new Error(`Entry "${targetName}" not found in ZIP`));
|
||||||
|
});
|
||||||
|
zipfile.on("error", reject);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch(reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect all entry paths from a ZIP archive using yauzl.
|
||||||
|
* Returns normalised forward-slash paths.
|
||||||
|
*/
|
||||||
|
function collectZipEntries(zipPath: string): Promise<string[]> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
import("yauzl")
|
||||||
|
.then((yauzl) => {
|
||||||
|
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
|
||||||
|
if (err || !zipfile) {
|
||||||
|
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const entries: string[] = [];
|
||||||
|
zipfile.readEntry();
|
||||||
|
zipfile.on("entry", (entry) => {
|
||||||
|
entries.push(entry.fileName.replace(/\\/g, "/"));
|
||||||
|
zipfile.readEntry();
|
||||||
|
});
|
||||||
|
zipfile.on("end", () => {
|
||||||
|
zipfile.close();
|
||||||
|
resolve(entries);
|
||||||
|
});
|
||||||
|
zipfile.on("error", reject);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch(reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract a single entry from a ZIP archive to a destination path.
|
||||||
|
*/
|
||||||
|
async function extractZipEntry(
|
||||||
|
zipPath: string,
|
||||||
|
entryName: string,
|
||||||
|
destPath: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const { entry, zipfile } = await openZipAndFindEntry(zipPath, entryName);
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
zipfile.openReadStream(entry, (streamErr, readStream) => {
|
||||||
|
if (streamErr || !readStream) {
|
||||||
|
zipfile.close();
|
||||||
|
reject(streamErr ?? new Error("Failed to open entry stream"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const writeStream = createWriteStream(destPath);
|
||||||
|
pipeline(readStream, writeStream)
|
||||||
|
.then(() => {
|
||||||
|
zipfile.close();
|
||||||
|
resolve();
|
||||||
|
})
|
||||||
|
.catch((pipelineError) => {
|
||||||
|
zipfile.close();
|
||||||
|
reject(pipelineError);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read media file references from the first N records of a JSONL file to verify
|
||||||
|
* that referenced files exist inside the ZIP.
|
||||||
|
*
|
||||||
|
* Collects from all known schema fields: `wav_fn` (audio), `img_path` +
|
||||||
|
* `input_img` (image generation), `first_frame_path` / `last_frame_path` /
|
||||||
|
* `video_path` (video generation), `image_fn` / `video_fn` (legacy).
|
||||||
|
*/
|
||||||
|
async function collectMediaRefs(
|
||||||
|
jsonlPath: string,
|
||||||
|
maxLines = 100,
|
||||||
|
): Promise<{ refs: string[]; totalLines: number }> {
|
||||||
|
const { createReadStream } = await import("fs");
|
||||||
|
const { createInterface } = await import("readline");
|
||||||
|
const stream = createReadStream(jsonlPath, { encoding: "utf8" });
|
||||||
|
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||||
|
const refs: string[] = [];
|
||||||
|
let totalLines = 0;
|
||||||
|
for await (const raw of rl) {
|
||||||
|
totalLines++;
|
||||||
|
if (refs.length >= maxLines) continue;
|
||||||
|
const line = raw.trim();
|
||||||
|
if (line.length === 0) continue;
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(line) as Record<string, unknown>;
|
||||||
|
if (typeof obj.wav_fn === "string") refs.push(obj.wav_fn);
|
||||||
|
if (typeof obj.img_path === "string") refs.push(obj.img_path);
|
||||||
|
if (typeof obj.input_img === "string") refs.push(obj.input_img);
|
||||||
|
if (typeof obj.first_frame_path === "string") refs.push(obj.first_frame_path);
|
||||||
|
if (typeof obj.last_frame_path === "string") refs.push(obj.last_frame_path);
|
||||||
|
if (typeof obj.video_path === "string") refs.push(obj.video_path);
|
||||||
|
if (typeof obj.image_fn === "string") refs.push(obj.image_fn);
|
||||||
|
if (typeof obj.video_fn === "string") refs.push(obj.video_fn);
|
||||||
|
} catch {
|
||||||
|
// Parse errors are reported by the JSONL validator, not here.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { refs, totalLines };
|
||||||
|
}
|
||||||
|
|
||||||
|
export const zipValidator: ValidatorSpec = {
|
||||||
|
format: "zip",
|
||||||
|
extensions: [".zip"],
|
||||||
|
|
||||||
|
async validate(filePath: string, opts: ValidateOpts): Promise<ValidationResult> {
|
||||||
|
const start = Date.now();
|
||||||
|
const errors: ValidationIssue[] = [];
|
||||||
|
const warnings: ValidationIssue[] = [];
|
||||||
|
|
||||||
|
// --- 1. Collect ZIP entries ---
|
||||||
|
let entries: string[];
|
||||||
|
try {
|
||||||
|
entries = await collectZipEntries(filePath);
|
||||||
|
} catch (openError) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
format: "zip",
|
||||||
|
filePath,
|
||||||
|
errors: [
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"ZIP_OPEN_FAILED",
|
||||||
|
`Could not open ZIP archive: ${(openError as Error).message}`,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
stats: { durationMs: Date.now() - start },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entries.length === 0) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
format: "zip",
|
||||||
|
filePath,
|
||||||
|
errors: [makeIssue("error", "ZIP_EMPTY", `ZIP archive contains no entries.`)],
|
||||||
|
warnings: [],
|
||||||
|
stats: { durationMs: Date.now() - start },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 2. Check for data.jsonl ---
|
||||||
|
const hasDataJsonl = entries.some(
|
||||||
|
(entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"),
|
||||||
|
);
|
||||||
|
if (!hasDataJsonl) {
|
||||||
|
errors.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"MISSING_DATA_JSONL",
|
||||||
|
`ZIP archive must contain "data.jsonl" at the root. ` +
|
||||||
|
`This file maps media files (e.g. .wav) to their labels.`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 3. Check for train/ directory (modality-aware) ---
|
||||||
|
// Audio TTS data uses a train/ subdirectory; image and video generation
|
||||||
|
// data do not (image is flat; video i2v is flat, kf2v uses image//video/).
|
||||||
|
// Only warn about missing train/ for audio (schema === "tts" or auto-detect
|
||||||
|
// when we don't know the schema yet).
|
||||||
|
const isImageSchema = opts.schema === "image";
|
||||||
|
const isVideoSchema = opts.schema === "video";
|
||||||
|
const hasTrainDir = entries.some((entry) => entry === "train/" || entry.startsWith("train/"));
|
||||||
|
// The train/ layout is an audio-TTS convention (media referenced as
|
||||||
|
// "train/xxx.wav"). It is only meaningful when the archive actually contains
|
||||||
|
// .wav files — image/video ZIPs (flat, or kf2v's image//video/ layout)
|
||||||
|
// legitimately have no train/ dir. Gating on .wav presence also fixes the
|
||||||
|
// auto-detect path (opts.schema undefined), where we would otherwise
|
||||||
|
// false-warn on every image/video archive.
|
||||||
|
const hasWavFiles = entries.some((entry) => entry.toLowerCase().endsWith(".wav"));
|
||||||
|
if (!hasTrainDir && !isImageSchema && !isVideoSchema && hasWavFiles) {
|
||||||
|
warnings.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"NO_TRAIN_DIR",
|
||||||
|
`No "train/" directory found in the ZIP. Media files are typically ` +
|
||||||
|
`placed under "train/" and referenced as "train/xxx.wav" in data.jsonl.`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 3b. Minimum image count check (image generation only) ---
|
||||||
|
// The platform requires at least 25 training images (50+ recommended).
|
||||||
|
if (isImageSchema) {
|
||||||
|
const MIN_IMAGES = 25;
|
||||||
|
const imageFiles = entries.filter((entry) => {
|
||||||
|
if (entry === "data.jsonl" || entry.endsWith("/data.jsonl")) return false;
|
||||||
|
if (entry.endsWith("/")) return false; // directory entries
|
||||||
|
const dot = entry.lastIndexOf(".");
|
||||||
|
const ext = dot >= 0 ? entry.slice(dot).toLowerCase() : "";
|
||||||
|
return IMAGE_EXTENSIONS.has(ext);
|
||||||
|
});
|
||||||
|
if (imageFiles.length < MIN_IMAGES) {
|
||||||
|
errors.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"INSUFFICIENT_IMAGES",
|
||||||
|
`Found ${imageFiles.length} image(s) in ZIP, but image generation fine-tuning ` +
|
||||||
|
`requires at least ${MIN_IMAGES} images (50+ recommended).`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If data.jsonl is missing, we can't do JSONL content validation.
|
||||||
|
if (!hasDataJsonl) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
format: "zip",
|
||||||
|
filePath,
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
stats: { totalRecords: entries.length, durationMs: Date.now() - start },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 4. Extract data.jsonl to a temp file and run jsonlValidator ---
|
||||||
|
const dataJsonlEntry = entries.find(
|
||||||
|
(entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"),
|
||||||
|
)!;
|
||||||
|
const tmpDir = join(tmpdir(), `bl-zip-${randomBytes(6).toString("hex")}`);
|
||||||
|
mkdirSync(tmpDir, { recursive: true });
|
||||||
|
const tmpJsonl = join(tmpDir, "data.jsonl");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await extractZipEntry(filePath, dataJsonlEntry, tmpJsonl);
|
||||||
|
} catch (extractError) {
|
||||||
|
errors.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"EXTRACT_FAILED",
|
||||||
|
`Failed to extract "data.jsonl" from ZIP: ${(extractError as Error).message}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
format: "zip",
|
||||||
|
filePath,
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
stats: { durationMs: Date.now() - start },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delegate JSONL content validation. The profile layer passes opts.schema
|
||||||
|
// (e.g. "tts") so the right record-schema spec is used.
|
||||||
|
const jsonlResult = await jsonlValidator.validate(tmpJsonl, opts);
|
||||||
|
errors.push(...jsonlResult.errors);
|
||||||
|
warnings.push(...jsonlResult.warnings);
|
||||||
|
|
||||||
|
// --- 5. Verify media file references (sample first 100 records) ---
|
||||||
|
if (jsonlResult.valid) {
|
||||||
|
const { refs } = await collectMediaRefs(tmpJsonl);
|
||||||
|
const entrySet = new Set(entries);
|
||||||
|
// Media paths in data.jsonl are relative to the manifest's location. Many
|
||||||
|
// official sample archives wrap everything in a single top-level folder
|
||||||
|
// (e.g. "wan-i2v-valid-dataset/data.jsonl" alongside
|
||||||
|
// "wan-i2v-valid-dataset/image_1.jpg"), so a bare "image_1.jpg" ref
|
||||||
|
// resolves against that folder, not the ZIP root. Derive the manifest's
|
||||||
|
// directory prefix and accept either the wrapped or root-relative form.
|
||||||
|
const slash = dataJsonlEntry.lastIndexOf("/");
|
||||||
|
const baseDir = slash >= 0 ? dataJsonlEntry.slice(0, slash + 1) : "";
|
||||||
|
const danglingRefs: string[] = [];
|
||||||
|
for (const ref of refs) {
|
||||||
|
// Normalise: some archives use "train/foo.wav", some use "./train/foo.wav".
|
||||||
|
const normalised = ref.replace(/^\.\//, "");
|
||||||
|
if (entrySet.has(normalised) || entrySet.has(baseDir + normalised)) continue;
|
||||||
|
danglingRefs.push(ref);
|
||||||
|
}
|
||||||
|
if (danglingRefs.length > 0) {
|
||||||
|
const shown = danglingRefs.slice(0, 5).join(", ");
|
||||||
|
const suffix = danglingRefs.length > 5 ? ` (and ${danglingRefs.length - 5} more)` : "";
|
||||||
|
errors.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"DANGLING_MEDIA_REFS",
|
||||||
|
`${danglingRefs.length} media file(s) referenced in data.jsonl not found in ZIP: ${shown}${suffix}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 6. Clean up ---
|
||||||
|
rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
|
||||||
|
return {
|
||||||
|
valid: errors.length === 0,
|
||||||
|
format: "zip",
|
||||||
|
filePath,
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
stats: {
|
||||||
|
totalRecords: jsonlResult.stats.totalRecords ?? entries.length,
|
||||||
|
sampledRecords: jsonlResult.stats.sampledRecords,
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -119,8 +119,8 @@ export interface CreateDeploymentRequest {
|
|||||||
plan: string;
|
plan: string;
|
||||||
/** Required by API even for token-billed (lora) plans where it is ignored — CLI injects 1. */
|
/** Required by API even for token-billed (lora) plans where it is ignored — CLI injects 1. */
|
||||||
capacity?: number;
|
capacity?: number;
|
||||||
/** Optional template id for advanced configurations. */
|
/** Deploy spec id (e.g. "MU1", "dps-..."), sent as `deploy_spec` in POST body. */
|
||||||
template_id?: string;
|
deploy_spec?: string;
|
||||||
/**
|
/**
|
||||||
* PTU capacity (provisioned throughput limits). Only effective when
|
* PTU capacity (provisioned throughput limits). Only effective when
|
||||||
* `plan === "ptu"`. The doc says this defaults to 10000/1000 when omitted,
|
* `plan === "ptu"`. The doc says this defaults to 10000/1000 when omitted,
|
||||||
@@ -128,10 +128,29 @@ export interface CreateDeploymentRequest {
|
|||||||
* info"), so the CLI treats it as required for ptu.
|
* info"), so the CLI treats it as required for ptu.
|
||||||
*/
|
*/
|
||||||
ptu_capacity?: PtuCapacity;
|
ptu_capacity?: PtuCapacity;
|
||||||
|
/**
|
||||||
|
* AIGC generation config for fine-tuned Wan video (i2v/kf2v) LoRA deployments.
|
||||||
|
* Ignored by non-video plans. See `AigcConfig`.
|
||||||
|
*/
|
||||||
|
aigc_config?: AigcConfig;
|
||||||
/** Future-compat: arbitrary additional fields are forwarded as-is. */
|
/** Future-compat: arbitrary additional fields are forwarded as-is. */
|
||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AIGC generation config — used when deploying fine-tuned Wan video (i2v/kf2v)
|
||||||
|
* LoRA models. Controls how prompts are applied at inference time:
|
||||||
|
* - use_input_prompt=false: ignore the caller's prompt, use the preset
|
||||||
|
* `prompt` template instead (the common LoRA case).
|
||||||
|
* - use_input_prompt=true: honor the caller's prompt.
|
||||||
|
* `lora_prompt_default` is the default trigger-word phrase appended for the LoRA.
|
||||||
|
*/
|
||||||
|
export interface AigcConfig {
|
||||||
|
use_input_prompt?: boolean;
|
||||||
|
prompt?: string;
|
||||||
|
lora_prompt_default?: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** PTU throughput limits — only used when `plan === "ptu"`. */
|
/** PTU throughput limits — only used when `plan === "ptu"`. */
|
||||||
export interface PtuCapacity {
|
export interface PtuCapacity {
|
||||||
/** Max input tokens per minute (all models). */
|
/** Max input tokens per minute (all models). */
|
||||||
|
|||||||
@@ -57,11 +57,6 @@ export function isTrainingTypeCli(value: string): value is TrainingTypeCli {
|
|||||||
return value in TRAINING_TYPE_MAP;
|
return value in TRAINING_TYPE_MAP;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Map a CLI training type to the server `training_type` for the request body. */
|
|
||||||
export function toServerTrainingType(value: TrainingTypeCli): string {
|
|
||||||
return TRAINING_TYPE_MAP[value].server;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The (method, variant) pair a CLI training type resolves to. */
|
/** The (method, variant) pair a CLI training type resolves to. */
|
||||||
export function trainingTypeMethodVariant(value: TrainingTypeCli): {
|
export function trainingTypeMethodVariant(value: TrainingTypeCli): {
|
||||||
method: string;
|
method: string;
|
||||||
|
|||||||
@@ -2,3 +2,4 @@ export * from "./types.ts";
|
|||||||
export * from "./api.ts";
|
export * from "./api.ts";
|
||||||
export * from "./capability.ts";
|
export * from "./capability.ts";
|
||||||
export * from "./preflight.ts";
|
export * from "./preflight.ts";
|
||||||
|
export * from "./profiles/index.ts";
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
/**
|
||||||
|
* Shared helpers for training profiles.
|
||||||
|
*
|
||||||
|
* Most text-based training types (sft, dpo, cpt, and their -lora variants)
|
||||||
|
* share the same hyper-parameter resolution logic: n_epochs defaults to 3,
|
||||||
|
* learning_rate and max_length are set only when explicitly provided, and
|
||||||
|
* batch_size is clamped to the server's [8, 1024] range. Extracting this
|
||||||
|
* into a single helper prevents drift when defaults or bounds change.
|
||||||
|
*/
|
||||||
|
import type { TrainingProfile, DataModality } from "./types.ts";
|
||||||
|
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
|
||||||
|
import type { DatasetSchema } from "../../dataset/validate/types.ts";
|
||||||
|
import { validateDataset } from "../../dataset/validate/registry.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve text-mode hyper-parameters from CLI flags.
|
||||||
|
*
|
||||||
|
* Shared by every profile's text branch (and by profiles that only support
|
||||||
|
* text). The audio branch in `sft-lora` uses its own `AUDIO_HYPER_PARAMS`.
|
||||||
|
*/
|
||||||
|
export function resolveTextHyperParameters(
|
||||||
|
flags: Record<string, unknown>,
|
||||||
|
): Record<string, unknown> {
|
||||||
|
const hp: Record<string, unknown> = {};
|
||||||
|
hp.n_epochs = flags.nEpochs !== undefined ? (flags.nEpochs as number) : 3;
|
||||||
|
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
|
||||||
|
if (flags.maxLength !== undefined) hp.max_length = flags.maxLength as number;
|
||||||
|
if (flags.batchSize !== undefined) {
|
||||||
|
const requested = flags.batchSize as number;
|
||||||
|
let batchSize = requested;
|
||||||
|
if (batchSize < 8) batchSize = 8;
|
||||||
|
if (batchSize > 1024) batchSize = 1024;
|
||||||
|
hp.batch_size = batchSize;
|
||||||
|
}
|
||||||
|
return hp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory for text-only training profiles (sft, dpo, dpo-lora, cpt).
|
||||||
|
*
|
||||||
|
* These profiles are structurally identical — they differ only in three
|
||||||
|
* string constants (CLI name, server name, record schema). Using a factory
|
||||||
|
* eliminates four near-identical files and prevents drift when the
|
||||||
|
* TrainingProfile interface changes.
|
||||||
|
*/
|
||||||
|
export function textProfile(
|
||||||
|
clientTrainingType: string,
|
||||||
|
serverTrainingType: string,
|
||||||
|
schema: DatasetSchema,
|
||||||
|
): TrainingProfile {
|
||||||
|
return {
|
||||||
|
clientTrainingType,
|
||||||
|
serverTrainingType,
|
||||||
|
acceptedExtensions: [".jsonl"],
|
||||||
|
|
||||||
|
async validate(
|
||||||
|
filePath: string,
|
||||||
|
_modality: DataModality,
|
||||||
|
opts: ValidateOpts,
|
||||||
|
): Promise<ValidationResult> {
|
||||||
|
return validateDataset(filePath, { ...opts, schema });
|
||||||
|
},
|
||||||
|
|
||||||
|
resolveHyperParameters(
|
||||||
|
_modality: DataModality,
|
||||||
|
flags: Record<string, unknown>,
|
||||||
|
): Record<string, unknown> {
|
||||||
|
return resolveTextHyperParameters(flags);
|
||||||
|
},
|
||||||
|
|
||||||
|
shouldSkipGate(_gate: string, _modality: DataModality): boolean {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
|
||||||
|
shouldSkipCapabilityCheck(_modality: DataModality): boolean {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/**
|
||||||
|
* `cpt` profile — Continual Pre-Training (full-parameter).
|
||||||
|
* Maps to the server's `cpt` training type. CPT record schema.
|
||||||
|
*/
|
||||||
|
import { textProfile } from "./common.ts";
|
||||||
|
|
||||||
|
export const cptProfile = textProfile("cpt", "cpt", "cpt");
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/**
|
||||||
|
* `dpo-lora` profile — LoRA variant of Direct Preference Optimization.
|
||||||
|
* Maps to the server's `dpo_lora` training type. DPO record schema.
|
||||||
|
*/
|
||||||
|
import { textProfile } from "./common.ts";
|
||||||
|
|
||||||
|
export const dpoLoraProfile = textProfile("dpo-lora", "dpo_lora", "dpo");
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/**
|
||||||
|
* `dpo` profile — Direct Preference Optimization (full-parameter).
|
||||||
|
* Maps to the server's `dpo_full` training type. DPO record schema.
|
||||||
|
*/
|
||||||
|
import { textProfile } from "./common.ts";
|
||||||
|
|
||||||
|
export const dpoProfile = textProfile("dpo", "dpo_full", "dpo");
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export type { TrainingProfile, DataModality } from "./types.ts";
|
||||||
|
export { getProfile, listTrainingTypes } from "./registry.ts";
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
/**
|
||||||
|
* Training profile registry — single point of truth for which training types
|
||||||
|
* the CLI supports.
|
||||||
|
*
|
||||||
|
* Routing: `--training-type <value>` → exact match on `clientTrainingType`.
|
||||||
|
* Unknown values are rejected with a USAGE error listing all registered types.
|
||||||
|
*
|
||||||
|
* Adding a new training type:
|
||||||
|
* 1. Create `<name>.ts` exporting a `TrainingProfile` constant.
|
||||||
|
* 2. Import and append to `PROFILES`.
|
||||||
|
* That's it. `create.ts` never needs to change.
|
||||||
|
*/
|
||||||
|
import { BailianError } from "../../errors/base.ts";
|
||||||
|
import { ExitCode } from "../../errors/codes.ts";
|
||||||
|
import type { TrainingProfile } from "./types.ts";
|
||||||
|
import { sftProfile } from "./sft.ts";
|
||||||
|
import { sftLoraProfile } from "./sft-lora.ts";
|
||||||
|
import { dpoProfile } from "./dpo.ts";
|
||||||
|
import { dpoLoraProfile } from "./dpo-lora.ts";
|
||||||
|
import { cptProfile } from "./cpt.ts";
|
||||||
|
|
||||||
|
const PROFILES: TrainingProfile[] = [
|
||||||
|
sftProfile,
|
||||||
|
sftLoraProfile,
|
||||||
|
dpoProfile,
|
||||||
|
dpoLoraProfile,
|
||||||
|
cptProfile,
|
||||||
|
];
|
||||||
|
|
||||||
|
/** Look up a profile by its CLI training-type name. Throws USAGE if unknown. */
|
||||||
|
export function getProfile(clientTrainingType: string): TrainingProfile {
|
||||||
|
const p = PROFILES.find((p) => p.clientTrainingType === clientTrainingType);
|
||||||
|
if (!p) {
|
||||||
|
const known = PROFILES.map((p) => p.clientTrainingType).join(", ");
|
||||||
|
throw new BailianError(
|
||||||
|
`Unknown training type "${clientTrainingType}".`,
|
||||||
|
ExitCode.USAGE,
|
||||||
|
`Supported training types: ${known}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** All registered CLI training-type names (for help text / whitelisting). */
|
||||||
|
export function listTrainingTypes(): string[] {
|
||||||
|
return PROFILES.map((p) => p.clientTrainingType);
|
||||||
|
}
|
||||||
|
|
||||||
|
export type { TrainingProfile } from "./types.ts";
|
||||||
|
export type { DataModality } from "./types.ts";
|
||||||
@@ -0,0 +1,231 @@
|
|||||||
|
/**
|
||||||
|
* `sft-lora` profile — LoRA fine-tuning via the server's `efficient_sft`.
|
||||||
|
*
|
||||||
|
* Accepts `.jsonl` (text data with ChatML `{messages}` schema) and `.zip`
|
||||||
|
* (audio / image data with a `data.jsonl` manifest inside). The data
|
||||||
|
* inspector detects the modality from file content; this profile routes to the
|
||||||
|
* correct validator and assembles modality-specific hyper-parameters.
|
||||||
|
*
|
||||||
|
* Text, audio, and image all share the same server training type
|
||||||
|
* (`efficient_sft`) but differ in every other dimension: validation rules,
|
||||||
|
* hyper-parameters, and pre-flight gates. All branching is internal —
|
||||||
|
* `create.ts` calls the uniform profile interface without knowing the modality.
|
||||||
|
*/
|
||||||
|
import type { TrainingProfile, DataModality } from "./types.ts";
|
||||||
|
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
|
||||||
|
import { validateDataset } from "../../dataset/validate/registry.ts";
|
||||||
|
import { makeIssue, MAX_MEDIA_ZIP_BYTES } from "../../dataset/validate/common.ts";
|
||||||
|
import { resolveTextHyperParameters } from "./common.ts";
|
||||||
|
|
||||||
|
/** Audio TTS hyper-parameter defaults (CosyVoice v3 Flash). */
|
||||||
|
const AUDIO_HYPER_PARAMS: Record<string, unknown> = {
|
||||||
|
lm_max_epoch: 60,
|
||||||
|
lm_step: 5,
|
||||||
|
lm_num: 3,
|
||||||
|
lm_batch_size: 1000,
|
||||||
|
fm_max_epoch: 100,
|
||||||
|
fm_step: 10,
|
||||||
|
fm_num: 3,
|
||||||
|
fm_batch_size: 2000,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Image generation hyper-parameter defaults (Wan2.x).
|
||||||
|
*
|
||||||
|
* The platform accepts these via `hyper_parameters` in the POST /fine-tunes
|
||||||
|
* body. `generation_type` controls `max_pixels` and `val_img_size` defaults:
|
||||||
|
* - t2i: "2k" (2048×2048)
|
||||||
|
* - i2i: "1k" (1024×1024)
|
||||||
|
*
|
||||||
|
* The default is "t2i". The generation type is auto-detected from data
|
||||||
|
* content: if the first JSONL record has `input_img`, it's I2I; otherwise T2I.
|
||||||
|
* The inspector returns `"image-i2i"` for I2I data, which this profile uses
|
||||||
|
* directly to pick the right hyper-parameter set.
|
||||||
|
* `split` (0.9) auto-splits training data into train/validation when no
|
||||||
|
* explicit validation_file_ids are provided.
|
||||||
|
*/
|
||||||
|
const IMAGE_HYPER_PARAMS_T2I: Record<string, unknown> = {
|
||||||
|
learning_rate: "3e-5",
|
||||||
|
max_steps: 800,
|
||||||
|
eval_steps: 200,
|
||||||
|
max_token_length: "1k",
|
||||||
|
gradient_clip: 0.5,
|
||||||
|
weight_decay: 0.02,
|
||||||
|
max_pixels: "2k",
|
||||||
|
val_img_size: "2k",
|
||||||
|
generation_type: "t2i",
|
||||||
|
lora_rank: 32,
|
||||||
|
save_total_limit: 10,
|
||||||
|
split: 0.9,
|
||||||
|
};
|
||||||
|
|
||||||
|
const IMAGE_HYPER_PARAMS_I2I: Record<string, unknown> = {
|
||||||
|
...IMAGE_HYPER_PARAMS_T2I,
|
||||||
|
max_pixels: "1k",
|
||||||
|
val_img_size: "1k",
|
||||||
|
generation_type: "i2i",
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Image / video ZIP size cap — shared constant from dataset/validate/common.ts.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Video generation hyper-parameter defaults (Wan i2v / kf2v).
|
||||||
|
*
|
||||||
|
* Shared across all video models; `batch_size` and `max_pixels` differ by model
|
||||||
|
* family (resolved per model in `resolveHyperParameters`):
|
||||||
|
* - wan2.5 (e.g. wan2.5-i2v-preview): batch_size 2, max_pixels 36864
|
||||||
|
* - wan2.2 (i2v-flash / kf2v-flash): batch_size 4, max_pixels 262144
|
||||||
|
*
|
||||||
|
* `learning_rate` is a string to avoid JSON-number precision loss (consistent
|
||||||
|
* with the image defaults). `split` (0.9) + `max_split_val_dataset_sample`
|
||||||
|
* (5) drive the automatic train/validation split when no explicit
|
||||||
|
* validation_file_ids are provided.
|
||||||
|
*/
|
||||||
|
const VIDEO_HYPER_PARAMS_BASE: Record<string, unknown> = {
|
||||||
|
n_epochs: 400,
|
||||||
|
learning_rate: "2e-5",
|
||||||
|
split: 0.9,
|
||||||
|
max_split_val_dataset_sample: 5,
|
||||||
|
eval_epochs: 50,
|
||||||
|
save_total_limit: 10,
|
||||||
|
lora_rank: 32,
|
||||||
|
lora_alpha: 32,
|
||||||
|
};
|
||||||
|
|
||||||
|
/** True for any image modality (base or subtype). */
|
||||||
|
function isImage(modality: DataModality): boolean {
|
||||||
|
return modality === "image" || modality === "image-i2i";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True for any video modality (i2v base or kf2v subtype). */
|
||||||
|
function isVideo(modality: DataModality): boolean {
|
||||||
|
return modality === "video" || modality === "video-kf2v";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** wan2.5 family uses smaller batch_size / max_pixels than wan2.2. */
|
||||||
|
function isWan25(model: string | undefined): boolean {
|
||||||
|
return typeof model === "string" && /wan2\.5/i.test(model);
|
||||||
|
}
|
||||||
|
|
||||||
|
export const sftLoraProfile: TrainingProfile = {
|
||||||
|
clientTrainingType: "sft-lora",
|
||||||
|
serverTrainingType: "efficient_sft",
|
||||||
|
acceptedExtensions: [".jsonl", ".zip"],
|
||||||
|
|
||||||
|
async validate(
|
||||||
|
filePath: string,
|
||||||
|
modality: DataModality,
|
||||||
|
opts: ValidateOpts,
|
||||||
|
): Promise<ValidationResult> {
|
||||||
|
if (modality === "audio") {
|
||||||
|
// ZIP validation: structure check + JSONL content via tts schema.
|
||||||
|
// The zip validator internally calls jsonlValidator for data.jsonl.
|
||||||
|
return validateDataset(filePath, { ...opts, schema: "tts" });
|
||||||
|
}
|
||||||
|
if (isImage(modality)) {
|
||||||
|
// Image generation: flat ZIP with data.jsonl + image files.
|
||||||
|
// The zip validator handles ≥25 image count + flat structure when
|
||||||
|
// schema is "image". Size cap is 1 GB (vs 300 MB for text/audio).
|
||||||
|
return validateDataset(filePath, {
|
||||||
|
...opts,
|
||||||
|
schema: "image",
|
||||||
|
maxBytes: MAX_MEDIA_ZIP_BYTES,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (isVideo(modality)) {
|
||||||
|
// Video generation (Wan i2v/kf2v): ZIP with data.jsonl + frame images and
|
||||||
|
// (for training) target videos. i2v is flat; kf2v uses image//video/
|
||||||
|
// subfolders. 1 GB cap like image.
|
||||||
|
const result = await validateDataset(filePath, {
|
||||||
|
...opts,
|
||||||
|
schema: "video",
|
||||||
|
maxBytes: MAX_MEDIA_ZIP_BYTES,
|
||||||
|
});
|
||||||
|
// Model <-> data cross-check: a kf2v model needs first+last-frame data,
|
||||||
|
// an i2v model ignores last_frame_path. Only runs when we know the model
|
||||||
|
// (bare file-id flows pass no model). Detected subtype comes from the
|
||||||
|
// data inspector (video = i2v, video-kf2v = has last_frame_path).
|
||||||
|
if (typeof opts.model === "string" && opts.model.length > 0) {
|
||||||
|
const modelIsKf2v = /kf2v/i.test(opts.model);
|
||||||
|
const dataIsKf2v = modality === "video-kf2v";
|
||||||
|
if (modelIsKf2v && !dataIsKf2v) {
|
||||||
|
result.errors.push(
|
||||||
|
makeIssue(
|
||||||
|
"error",
|
||||||
|
"KF2V_DATA_MISMATCH",
|
||||||
|
`Model "${opts.model}" is a first+last-frame (kf2v) model but the data has ` +
|
||||||
|
`no "last_frame_path". kf2v training data must include a last frame per record.`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else if (!modelIsKf2v && dataIsKf2v) {
|
||||||
|
result.warnings.push(
|
||||||
|
makeIssue(
|
||||||
|
"warning",
|
||||||
|
"I2V_LAST_FRAME_IGNORED",
|
||||||
|
`Model "${opts.model}" is a first-frame (i2v) model but the data includes ` +
|
||||||
|
`"last_frame_path"; the last frame will be ignored during training.`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.valid = result.errors.length === 0;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
// Text: standard JSONL validation with chatml schema.
|
||||||
|
return validateDataset(filePath, { ...opts, schema: "chatml" });
|
||||||
|
},
|
||||||
|
|
||||||
|
resolveHyperParameters(
|
||||||
|
modality: DataModality,
|
||||||
|
flags: Record<string, unknown>,
|
||||||
|
): Record<string, unknown> {
|
||||||
|
if (modality === "audio") {
|
||||||
|
// Audio: use TTS defaults, user flags override (MVP: all hardcoded).
|
||||||
|
return { ...AUDIO_HYPER_PARAMS };
|
||||||
|
}
|
||||||
|
if (isImage(modality)) {
|
||||||
|
// Image: modality "image-i2i" (auto-detected from data having input_img)
|
||||||
|
// uses I2I defaults; plain "image" uses T2I defaults.
|
||||||
|
const base = modality === "image-i2i" ? IMAGE_HYPER_PARAMS_I2I : IMAGE_HYPER_PARAMS_T2I;
|
||||||
|
const hp = { ...base };
|
||||||
|
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
|
||||||
|
return hp;
|
||||||
|
}
|
||||||
|
if (isVideo(modality)) {
|
||||||
|
// Video: shared defaults + model-family-specific batch_size / max_pixels.
|
||||||
|
// wan2.5 uses batch_size 2 / max_pixels 36864; wan2.2 uses 4 / 262144.
|
||||||
|
const wan25 = isWan25(flags.model as string | undefined);
|
||||||
|
const hp: Record<string, unknown> = {
|
||||||
|
...VIDEO_HYPER_PARAMS_BASE,
|
||||||
|
batch_size: wan25 ? 2 : 4,
|
||||||
|
max_pixels: wan25 ? 36864 : 262144,
|
||||||
|
};
|
||||||
|
// Optional overrides (no clamping — video batch_size is intentionally small).
|
||||||
|
if (flags.nEpochs !== undefined) hp.n_epochs = flags.nEpochs as number;
|
||||||
|
if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string;
|
||||||
|
return hp;
|
||||||
|
}
|
||||||
|
// Text: existing hyper-parameter logic (n_epochs, batch_size, etc.).
|
||||||
|
return resolveTextHyperParameters(flags);
|
||||||
|
},
|
||||||
|
|
||||||
|
shouldSkipGate(gate: string, modality: DataModality): boolean {
|
||||||
|
if (modality === "audio" || isImage(modality) || isVideo(modality)) {
|
||||||
|
// Audio has no batch_size hyper-parameter; image uses max_steps; video
|
||||||
|
// datasets are intentionally small (batch_size 2/4). All skip the
|
||||||
|
// batch-size pre-flight gate to avoid false rejections.
|
||||||
|
if (gate === "batch_size") return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
|
||||||
|
shouldSkipCapabilityCheck(modality: DataModality): boolean {
|
||||||
|
// Audio models (e.g. cosyvoice-v3-flash), image models
|
||||||
|
// (e.g. wan2.7-image-pro) and video models (e.g. wan2.5-i2v-preview) report
|
||||||
|
// supports.sft=false in listFoundationModels but the API accepts
|
||||||
|
// efficient_sft — skip to avoid false blocking.
|
||||||
|
return modality === "audio" || isImage(modality) || isVideo(modality);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/**
|
||||||
|
* `sft` profile — full-parameter Supervised Fine-Tuning.
|
||||||
|
* Maps to the server's `sft` training type. ChatML record schema.
|
||||||
|
*/
|
||||||
|
import { textProfile } from "./common.ts";
|
||||||
|
|
||||||
|
export const sftProfile = textProfile("sft", "sft", "chatml");
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
/**
|
||||||
|
* Training profile — single source of truth for "how a training type behaves".
|
||||||
|
*
|
||||||
|
* Each profile encapsulates everything the CLI needs to know about a specific
|
||||||
|
* `--training-type` value: what file formats it accepts, how to validate data,
|
||||||
|
* which hyper-parameters to use, which pre-flight gates to run, and whether
|
||||||
|
* the model-capability check should be skipped.
|
||||||
|
*
|
||||||
|
* Profiles are **decoupled from validators**: the profile declares which file
|
||||||
|
* extensions it accepts, and the data inspector (`dataset/inspect.ts`) detects
|
||||||
|
* the data modality by parsing the file content. The profile then routes to
|
||||||
|
* the appropriate validator internally — `create.ts` never sees an if-else
|
||||||
|
* about modalities or file formats.
|
||||||
|
*
|
||||||
|
* Adding a new training type = one new profile file + one line in the registry.
|
||||||
|
* Nothing in `create.ts` needs to change.
|
||||||
|
*/
|
||||||
|
import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Data modality detected by the inspector from file content.
|
||||||
|
*
|
||||||
|
* The inspector peeks at the first record of a JSONL or the `data.jsonl` inside
|
||||||
|
* a ZIP to determine what kind of data the file carries. This is orthogonal to
|
||||||
|
* the training type — `sft-lora` accepts both `.jsonl` (text) and `.zip`
|
||||||
|
* (audio / image / video).
|
||||||
|
*
|
||||||
|
* `"image-i2i"` is a subtype of `"image"` — the first record contains an
|
||||||
|
* `input_img` field (image-to-image). `"video-kf2v"` is a subtype of `"video"`
|
||||||
|
* — the first record contains a `last_frame_path` field (first+last-frame video,
|
||||||
|
* Wan kf2v). Profiles can branch on subtypes to adjust hyper-parameter defaults
|
||||||
|
* or cross-check the chosen `--model`. Callers that don't care about a subtype
|
||||||
|
* can normalise `"image-i2i"` → `"image"` and `"video-kf2v"` → `"video"`.
|
||||||
|
*/
|
||||||
|
export type DataModality = "text" | "audio" | "image" | "image-i2i" | "video" | "video-kf2v";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A training profile. One per `--training-type` CLI value.
|
||||||
|
*
|
||||||
|
* The profile owns all modality-specific branching internally — callers
|
||||||
|
* (`create.ts`) interact with it through a uniform interface and never need
|
||||||
|
* to know whether the data is text, audio, or something else.
|
||||||
|
*/
|
||||||
|
export interface TrainingProfile {
|
||||||
|
/** CLI vocabulary: the value users pass to `--training-type`. */
|
||||||
|
clientTrainingType: string;
|
||||||
|
|
||||||
|
/** Server `training_type` for the POST /fine-tunes request body. */
|
||||||
|
serverTrainingType: string;
|
||||||
|
|
||||||
|
/** File extensions this profile accepts (lower-case, with dot). */
|
||||||
|
acceptedExtensions: string[];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate the training data file. The profile internally routes to the
|
||||||
|
* correct validator based on `modality` (detected by the data inspector).
|
||||||
|
*/
|
||||||
|
validate(filePath: string, modality: DataModality, opts: ValidateOpts): Promise<ValidationResult>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the `hyper_parameters` object for the API request. Merges user-supplied
|
||||||
|
* flags with modality-specific defaults (e.g. audio TTS uses `lm_max_epoch` /
|
||||||
|
* `fm_max_epoch`, text SFT uses `n_epochs` / `batch_size`).
|
||||||
|
*/
|
||||||
|
resolveHyperParameters(
|
||||||
|
modality: DataModality,
|
||||||
|
flags: Record<string, unknown>,
|
||||||
|
): Record<string, unknown>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether a specific pre-flight gate should be skipped for the given modality.
|
||||||
|
* Common gates: `"batch_size"` (samples must exceed batch_size), etc.
|
||||||
|
* Audio TTS skips the batch-size gate (no batch_size hyper-parameter).
|
||||||
|
*/
|
||||||
|
shouldSkipGate(gate: string, modality: DataModality): boolean;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the model-capability pre-flight check should be skipped.
|
||||||
|
* Audio models (e.g. cosyvoice-v3-flash) report `supports.sft = false` in
|
||||||
|
* `listFoundationModels` but the API accepts `efficient_sft` — the check
|
||||||
|
* would incorrectly block the submission.
|
||||||
|
*/
|
||||||
|
shouldSkipCapabilityCheck(modality: DataModality): boolean;
|
||||||
|
}
|
||||||
@@ -169,10 +169,13 @@ describe("parseDatasetSchemaFlag", () => {
|
|||||||
expect(parseDatasetSchemaFlag(" ")).toBeUndefined();
|
expect(parseDatasetSchemaFlag(" ")).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("chatml / dpo / cpt pass through", () => {
|
test("chatml / dpo / cpt / tts / image / video pass through", () => {
|
||||||
expect(parseDatasetSchemaFlag("chatml")).toBe("chatml");
|
expect(parseDatasetSchemaFlag("chatml")).toBe("chatml");
|
||||||
expect(parseDatasetSchemaFlag("dpo")).toBe("dpo");
|
expect(parseDatasetSchemaFlag("dpo")).toBe("dpo");
|
||||||
expect(parseDatasetSchemaFlag("cpt")).toBe("cpt");
|
expect(parseDatasetSchemaFlag("cpt")).toBe("cpt");
|
||||||
|
expect(parseDatasetSchemaFlag("tts")).toBe("tts");
|
||||||
|
expect(parseDatasetSchemaFlag("image")).toBe("image");
|
||||||
|
expect(parseDatasetSchemaFlag("video")).toBe("video");
|
||||||
expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo");
|
expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Generated
+33
-3
@@ -9,6 +9,9 @@ catalogs:
|
|||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^24
|
specifier: ^24
|
||||||
version: 24.12.2
|
version: 24.12.2
|
||||||
|
'@types/yauzl':
|
||||||
|
specifier: ^3.4.0
|
||||||
|
version: 3.4.0
|
||||||
ajv:
|
ajv:
|
||||||
specifier: ^8.20.0
|
specifier: ^8.20.0
|
||||||
version: 8.20.0
|
version: 8.20.0
|
||||||
@@ -27,6 +30,9 @@ catalogs:
|
|||||||
yaml:
|
yaml:
|
||||||
specifier: ^2.8.3
|
specifier: ^2.8.3
|
||||||
version: 2.8.3
|
version: 2.8.3
|
||||||
|
yauzl:
|
||||||
|
specifier: ^3.4.0
|
||||||
|
version: 3.4.0
|
||||||
|
|
||||||
overrides:
|
overrides:
|
||||||
vite: npm:@voidzero-dev/vite-plus-core@latest
|
vite: npm:@voidzero-dev/vite-plus-core@latest
|
||||||
@@ -119,10 +125,16 @@ importers:
|
|||||||
yaml:
|
yaml:
|
||||||
specifier: ^2.8.3
|
specifier: ^2.8.3
|
||||||
version: 2.8.3
|
version: 2.8.3
|
||||||
|
yauzl:
|
||||||
|
specifier: 'catalog:'
|
||||||
|
version: 3.4.0
|
||||||
devDependencies:
|
devDependencies:
|
||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: 'catalog:'
|
specifier: 'catalog:'
|
||||||
version: 24.12.2
|
version: 24.12.2
|
||||||
|
'@types/yauzl':
|
||||||
|
specifier: 'catalog:'
|
||||||
|
version: 3.4.0
|
||||||
'@typescript/native-preview':
|
'@typescript/native-preview':
|
||||||
specifier: 7.0.0-dev.20260328.1
|
specifier: 7.0.0-dev.20260328.1
|
||||||
version: 7.0.0-dev.20260328.1
|
version: 7.0.0-dev.20260328.1
|
||||||
@@ -645,6 +657,9 @@ packages:
|
|||||||
'@types/node@25.6.0':
|
'@types/node@25.6.0':
|
||||||
resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}
|
resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==}
|
||||||
|
|
||||||
|
'@types/yauzl@3.4.0':
|
||||||
|
resolution: {integrity: sha512-NRPn5w6h8dhcnmx3YIRQcqMywY/+nND/uOkJessedcrowO3C0AssHp3tMJpxKAwOhFOo0OV1y9VtsC5hbKKBAw==}
|
||||||
|
|
||||||
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
|
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
|
||||||
resolution: {integrity: sha512-BmJGDWC0bSQ2w5O/E+Mw9eTv9RklJ3vjshu7UdD92bUMxc4V4dkBhYj5r0qxbl4f+VFNX7fXvcDDI+9o+Kb6yw==}
|
resolution: {integrity: sha512-BmJGDWC0bSQ2w5O/E+Mw9eTv9RklJ3vjshu7UdD92bUMxc4V4dkBhYj5r0qxbl4f+VFNX7fXvcDDI+9o+Kb6yw==}
|
||||||
cpu: [arm64]
|
cpu: [arm64]
|
||||||
@@ -1021,6 +1036,9 @@ packages:
|
|||||||
oxlint-tsgolint:
|
oxlint-tsgolint:
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
pend@1.2.0:
|
||||||
|
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
|
||||||
|
|
||||||
picocolors@1.1.1:
|
picocolors@1.1.1:
|
||||||
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
|
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
|
||||||
|
|
||||||
@@ -1193,6 +1211,10 @@ packages:
|
|||||||
engines: {node: '>= 14.6'}
|
engines: {node: '>= 14.6'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
yauzl@3.4.0:
|
||||||
|
resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==}
|
||||||
|
engines: {node: '>=12'}
|
||||||
|
|
||||||
snapshots:
|
snapshots:
|
||||||
|
|
||||||
'@clack/core@0.3.5':
|
'@clack/core@0.3.5':
|
||||||
@@ -1443,7 +1465,10 @@ snapshots:
|
|||||||
'@types/node@25.6.0':
|
'@types/node@25.6.0':
|
||||||
dependencies:
|
dependencies:
|
||||||
undici-types: 7.19.2
|
undici-types: 7.19.2
|
||||||
optional: true
|
|
||||||
|
'@types/yauzl@3.4.0':
|
||||||
|
dependencies:
|
||||||
|
'@types/node': 25.6.0
|
||||||
|
|
||||||
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
|
'@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1':
|
||||||
optional: true
|
optional: true
|
||||||
@@ -1781,6 +1806,8 @@ snapshots:
|
|||||||
'@oxlint/binding-win32-x64-msvc': 1.63.0
|
'@oxlint/binding-win32-x64-msvc': 1.63.0
|
||||||
oxlint-tsgolint: 0.22.1
|
oxlint-tsgolint: 0.22.1
|
||||||
|
|
||||||
|
pend@1.2.0: {}
|
||||||
|
|
||||||
picocolors@1.1.1: {}
|
picocolors@1.1.1: {}
|
||||||
|
|
||||||
picomatch@4.0.4: {}
|
picomatch@4.0.4: {}
|
||||||
@@ -1874,8 +1901,7 @@ snapshots:
|
|||||||
|
|
||||||
undici-types@7.16.0: {}
|
undici-types@7.16.0: {}
|
||||||
|
|
||||||
undici-types@7.19.2:
|
undici-types@7.19.2: {}
|
||||||
optional: true
|
|
||||||
|
|
||||||
undici@8.4.1: {}
|
undici@8.4.1: {}
|
||||||
|
|
||||||
@@ -2016,3 +2042,7 @@ snapshots:
|
|||||||
ws@8.20.0: {}
|
ws@8.20.0: {}
|
||||||
|
|
||||||
yaml@2.8.3: {}
|
yaml@2.8.3: {}
|
||||||
|
|
||||||
|
yauzl@3.4.0:
|
||||||
|
dependencies:
|
||||||
|
pend: 1.2.0
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ packages:
|
|||||||
|
|
||||||
catalog:
|
catalog:
|
||||||
"@types/node": ^24
|
"@types/node": ^24
|
||||||
|
"@types/yauzl": ^3.4.0
|
||||||
ajv: ^8.20.0
|
ajv: ^8.20.0
|
||||||
boxen: ^8.0.1
|
boxen: ^8.0.1
|
||||||
chalk: ^5.6.2
|
chalk: ^5.6.2
|
||||||
@@ -13,6 +14,7 @@ catalog:
|
|||||||
vite-plus: latest
|
vite-plus: latest
|
||||||
vitest: npm:@voidzero-dev/vite-plus-test@latest
|
vitest: npm:@voidzero-dev/vite-plus-test@latest
|
||||||
yaml: ^2.8.3
|
yaml: ^2.8.3
|
||||||
|
yauzl: ^3.4.0
|
||||||
|
|
||||||
catalogMode: prefer
|
catalogMode: prefer
|
||||||
overrides:
|
overrides:
|
||||||
|
|||||||
@@ -7,13 +7,13 @@ Index: [index.md](index.md)
|
|||||||
|
|
||||||
## Commands in this group
|
## Commands in this group
|
||||||
|
|
||||||
| Command | Description |
|
| Command | Description |
|
||||||
| --------------------- | ---------------------------------------------------------- |
|
| --------------------- | ------------------------------------------------------------------ |
|
||||||
| `bl dataset delete` | Delete a dataset file by ID |
|
| `bl dataset delete` | Delete a dataset file by ID |
|
||||||
| `bl dataset get` | Get details of a single dataset file |
|
| `bl dataset get` | Get details of a single dataset file |
|
||||||
| `bl dataset list` | List uploaded dataset files |
|
| `bl dataset list` | List uploaded dataset files |
|
||||||
| `bl dataset upload` | Upload a dataset file (.jsonl) to Bailian |
|
| `bl dataset upload` | Upload a dataset file (.jsonl or .zip) to Bailian |
|
||||||
| `bl dataset validate` | Locally validate a dataset file (.jsonl) without uploading |
|
| `bl dataset validate` | Locally validate a dataset file (.jsonl or .zip) without uploading |
|
||||||
|
|
||||||
## Command details
|
## Command details
|
||||||
|
|
||||||
@@ -107,36 +107,36 @@ bl dataset list --output json
|
|||||||
|
|
||||||
### `bl dataset upload`
|
### `bl dataset upload`
|
||||||
|
|
||||||
| Field | Value |
|
| Field | Value |
|
||||||
| --------------- | -------------------------------------------------------------------------------------------------------------------- |
|
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| **Name** | `dataset upload` |
|
| **Name** | `dataset upload` |
|
||||||
| **Description** | Upload a dataset file (.jsonl) to Bailian |
|
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
|
||||||
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt>] [--no-validate] [--full-validate]` |
|
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
|
||||||
|
|
||||||
#### Flags
|
#### Flags
|
||||||
|
|
||||||
| Flag | Type | Required | Description |
|
| Flag | Type | Required | Description |
|
||||||
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------- |
|
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
| `--file <path>` | string | yes | Local .jsonl dataset file (≤300MB) |
|
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video) |
|
||||||
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
||||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record. |
|
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
|
||||||
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
|
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
|
||||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||||
| `--api-key <key>` | string | no | API key |
|
| `--api-key <key>` | string | no | API key |
|
||||||
| `--base-url <url>` | string | no | API base URL |
|
| `--base-url <url>` | string | no | API base URL |
|
||||||
|
|
||||||
#### Notes
|
#### Notes
|
||||||
|
|
||||||
- Only .jsonl is supported in this release. Three record schemas are
|
- Supports .jsonl (text) and .zip (audio/image/video archives with a
|
||||||
- recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],
|
- data.jsonl manifest). Six record schemas are recognized: chatml =
|
||||||
- chosen, rejected} where chosen/rejected are single assistant messages;
|
- {messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};
|
||||||
- cpt = {text:"..."} (continual pre-training, raw text). With no --schema,
|
- cpt = {text:"..."} (continual pre-training, raw text); tts =
|
||||||
- a record carrying chosen/rejected is validated as DPO, one with text (and
|
- {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =
|
||||||
- no messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to
|
- {img_path:"..."} (image generation); video = {first_frame_path:"...",
|
||||||
- require that shape on every record, or --schema chatml to ignore the
|
- video_path:"..."} (video generation). With no --schema, a record
|
||||||
- preference / text fields. Other purposes may carry a different schema in
|
- carrying wav_fn is validated as TTS, img_path as image, video_path /
|
||||||
- the future and would be served by a purpose-specific validator.
|
- first_frame_path as video, chosen/rejected as DPO, text (no messages)
|
||||||
- The dataset upload cap is 300MB per file.
|
- as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.
|
||||||
- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so
|
- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so
|
||||||
- the purpose tag is persisted (the DashScope-native /api/v1/files drops it).
|
- the purpose tag is persisted (the DashScope-native /api/v1/files drops it).
|
||||||
|
|
||||||
@@ -154,6 +154,10 @@ bl dataset upload --file dpo.jsonl --schema dpo
|
|||||||
bl dataset upload --file cpt.jsonl --schema cpt
|
bl dataset upload --file cpt.jsonl --schema cpt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bl dataset upload --file audio.zip --schema tts
|
||||||
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bl dataset upload --file eval.jsonl --purpose evaluation
|
bl dataset upload --file eval.jsonl --purpose evaluation
|
||||||
```
|
```
|
||||||
@@ -168,31 +172,35 @@ bl dataset upload --file train.jsonl --no-validate
|
|||||||
|
|
||||||
### `bl dataset validate`
|
### `bl dataset validate`
|
||||||
|
|
||||||
| Field | Value |
|
| Field | Value |
|
||||||
| --------------- | ----------------------------------------------------------------------------------- |
|
| --------------- | ------------------------------------------------------------------------------------------------------ |
|
||||||
| **Name** | `dataset validate` |
|
| **Name** | `dataset validate` |
|
||||||
| **Description** | Locally validate a dataset file (.jsonl) without uploading |
|
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
|
||||||
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt>]` |
|
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
|
||||||
|
|
||||||
#### Flags
|
#### Flags
|
||||||
|
|
||||||
| Flag | Type | Required | Description |
|
| Flag | Type | Required | Description |
|
||||||
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------- |
|
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
| `--file <path>` | string | yes | Local .jsonl dataset file |
|
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
|
||||||
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
|
||||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), or "cpt" (raw text). Default auto-detects per record. |
|
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
|
||||||
|
|
||||||
#### Notes
|
#### Notes
|
||||||
|
|
||||||
- Default scan: every line gets a structural check, then ~160 lines (front 50,
|
- Default scan: every line gets a structural check, then ~160 lines (front 50,
|
||||||
- evenly spaced 100, last 10) are JSON.parsed against the active schema.
|
- evenly spaced 100, last 10) are JSON.parsed against the active schema.
|
||||||
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
|
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
|
||||||
- rejected} where chosen/rejected are single assistant messages; cpt =
|
- rejected}; cpt = {text:"..."} (continual pre-training, raw text);
|
||||||
- {text:"..."} (continual pre-training, raw text). With no --schema, a
|
- tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
|
||||||
- record carrying chosen/rejected is validated as DPO, one with text (and no
|
- image = {img_path:"..."} (image generation); video =
|
||||||
- messages) as CPT, otherwise as ChatML. Pass --schema dpo / cpt to require
|
- {first_frame_path:"...", video_path:"..."} (video generation). With no
|
||||||
- that shape on every record (strict), or --schema chatml to ignore the
|
- --schema, a record carrying wav_fn is validated as TTS, img_path as
|
||||||
- preference / text fields. Use --full-validate to JSON.parse every line.
|
- image, video_path / first_frame_path as video, chosen/rejected as DPO,
|
||||||
|
- text (no messages) as CPT, otherwise ChatML. Pass --schema to require a
|
||||||
|
- specific shape on every record. ZIP archives (.zip) are validated
|
||||||
|
- structurally (data.jsonl present, media references resolve) in addition
|
||||||
|
- to per-record content checks. Use --full-validate to JSON.parse every line.
|
||||||
|
|
||||||
#### Examples
|
#### Examples
|
||||||
|
|
||||||
@@ -208,6 +216,10 @@ bl dataset validate --file dpo.jsonl --schema dpo
|
|||||||
bl dataset validate --file cpt.jsonl --schema cpt
|
bl dataset validate --file cpt.jsonl --schema cpt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bl dataset validate --file audio.zip --schema tts
|
||||||
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bl dataset validate --file eval.jsonl --full-validate
|
bl dataset validate --file eval.jsonl --full-validate
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -25,23 +25,26 @@ Index: [index.md](index.md)
|
|||||||
| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| **Name** | `deploy create` |
|
| **Name** | `deploy create` |
|
||||||
| **Description** | Create a model deployment |
|
| **Description** | Create a model deployment |
|
||||||
| **Usage** | `bl deploy create --model <model_name> --name <display_name> [--plan <plan>] [--template-id <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]` |
|
| **Usage** | `bl deploy create --model <model_name> --name <display_name> [--plan <plan>] [--deploy-spec <id>] [--capacity <n>] [--billing-method <m>] [--input-tpm <n>] [--output-tpm <n>] [--thinking-output-tpm <n>]` |
|
||||||
|
|
||||||
#### Flags
|
#### Flags
|
||||||
|
|
||||||
| Flag | Type | Required | Description |
|
| Flag | Type | Required | Description |
|
||||||
| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- |
|
| ----------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------ |
|
||||||
| `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) |
|
| `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) |
|
||||||
| `--name <display_name>` | string | yes | Console display name for the deployment (required) |
|
| `--name <display_name>` | string | yes | Console display name for the deployment (required) |
|
||||||
| `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu |
|
| `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu |
|
||||||
| `--template-id <id>` | string | no | Template id (only used by plan=mu; auto-picked if omitted) |
|
| `--deploy-spec <id>` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) |
|
||||||
| `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) |
|
| `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) |
|
||||||
| `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) |
|
| `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) |
|
||||||
| `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) |
|
| `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) |
|
||||||
| `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) |
|
| `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) |
|
||||||
| `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) |
|
| `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) |
|
||||||
| `--api-key <key>` | string | no | API key |
|
| `--aigc-use-input-prompt <bool>` | boolean | no | Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template) |
|
||||||
| `--base-url <url>` | string | no | API base URL |
|
| `--aigc-prompt <text>` | string | no | Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false |
|
||||||
|
| `--aigc-lora-prompt-default <text>` | string | no | Video LoRA (aigc_config): default trigger-word phrase for the LoRA |
|
||||||
|
| `--api-key <key>` | string | no | API key |
|
||||||
|
| `--base-url <url>` | string | no | API base URL |
|
||||||
|
|
||||||
#### Notes
|
#### Notes
|
||||||
|
|
||||||
@@ -49,12 +52,15 @@ Index: [index.md](index.md)
|
|||||||
- For plan=ptu (Token-billed, provisioned throughput), --input-tpm and
|
- For plan=ptu (Token-billed, provisioned throughput), --input-tpm and
|
||||||
- --output-tpm are required (the platform rejects creation without an
|
- --output-tpm are required (the platform rejects creation without an
|
||||||
- explicit ptu_capacity despite the doc listing defaults).
|
- explicit ptu_capacity despite the doc listing defaults).
|
||||||
- For plan=mu, `capacity`, `billing_method` and `template_id` are required.
|
- For plan=mu, `capacity`, `billing_method` and `deploy_spec` are required.
|
||||||
- billing_method defaults to POST_PAY (only supported value); template_id
|
- billing_method defaults to POST_PAY (only supported value); deploy_spec
|
||||||
- and capacity are auto-picked from GET /deployments/models when omitted.
|
- and capacity are auto-picked from GET /deployments/models when omitted.
|
||||||
- Use `bl deploy models --source base` to inspect available templates.
|
- Use `bl deploy models --source base` to inspect available templates.
|
||||||
- After creation, status starts at PENDING and transitions to RUNNING.
|
- After creation, status starts at PENDING and transitions to RUNNING.
|
||||||
- Invoke the deployed model with: bl text chat --model <deployed_model>
|
- Invoke the deployed model with: bl text chat --model <deployed_model>
|
||||||
|
- For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass
|
||||||
|
- --aigc-prompt / --aigc-lora-prompt-default (and optionally
|
||||||
|
- --aigc-use-input-prompt) to set the deployment's aigc_config.
|
||||||
- WARNING: --model is overloaded across commands and refers to DIFFERENT
|
- WARNING: --model is overloaded across commands and refers to DIFFERENT
|
||||||
- values. `bl deploy create --model` takes the exported model_name (e.g.
|
- values. `bl deploy create --model` takes the exported model_name (e.g.
|
||||||
- `qwen3-8b-ft-...`), but the create response also returns a `deployed_model`
|
- `qwen3-8b-ft-...`), but the create response also returns a `deployed_model`
|
||||||
@@ -78,7 +84,11 @@ bl deploy create --model qwen3-8b --name my-qwen3-mu --plan mu
|
|||||||
```
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --template-id MU1 --capacity 2
|
bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bl deploy create --model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."
|
||||||
```
|
```
|
||||||
|
|
||||||
### `bl deploy delete`
|
### `bl deploy delete`
|
||||||
|
|||||||
@@ -22,8 +22,8 @@ Use this index for the full quick index and global flags.
|
|||||||
| `bl dataset delete` | Delete a dataset file by ID | [dataset.md](dataset.md) |
|
| `bl dataset delete` | Delete a dataset file by ID | [dataset.md](dataset.md) |
|
||||||
| `bl dataset get` | Get details of a single dataset file | [dataset.md](dataset.md) |
|
| `bl dataset get` | Get details of a single dataset file | [dataset.md](dataset.md) |
|
||||||
| `bl dataset list` | List uploaded dataset files | [dataset.md](dataset.md) |
|
| `bl dataset list` | List uploaded dataset files | [dataset.md](dataset.md) |
|
||||||
| `bl dataset upload` | Upload a dataset file (.jsonl) to Bailian | [dataset.md](dataset.md) |
|
| `bl dataset upload` | Upload a dataset file (.jsonl or .zip) to Bailian | [dataset.md](dataset.md) |
|
||||||
| `bl dataset validate` | Locally validate a dataset file (.jsonl) without uploading | [dataset.md](dataset.md) |
|
| `bl dataset validate` | Locally validate a dataset file (.jsonl or .zip) without uploading | [dataset.md](dataset.md) |
|
||||||
| `bl deploy create` | Create a model deployment | [deploy.md](deploy.md) |
|
| `bl deploy create` | Create a model deployment | [deploy.md](deploy.md) |
|
||||||
| `bl deploy delete` | Delete a model deployment (must be STOPPED or FAILED) | [deploy.md](deploy.md) |
|
| `bl deploy delete` | Delete a model deployment (must be STOPPED or FAILED) | [deploy.md](deploy.md) |
|
||||||
| `bl deploy get` | Get details of a single model deployment | [deploy.md](deploy.md) |
|
| `bl deploy get` | Get details of a single model deployment | [deploy.md](deploy.md) |
|
||||||
|
|||||||
Reference in New Issue
Block a user