skills/modelstudioai__cli
1
0
Fork 0
mirror of https://github.com/modelstudioai/cli.git synced 2026-09-14 19:49:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add image/audio finetune

Browse Source
This commit is contained in:
故璃
2026-07-09 15:36:32 +08:00
parent 8fd072bcd1
commit d8aa89dc6c
14 changed files with 256 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/tests/e2e/dataset.e2e.test.ts
+54 -23
View File
@@ -216,7 +216,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => {
});
test("dataset upload --schema image --no-validate --dry-run 采用 1GB 媒体上限", async () => {
// image/video schemas raise the upload cap to 1 GiB (vs 300 MB for text).
// image schema raises the upload cap to 1 GiB (vs 300 MB for text).
// --no-validate keeps this offline (the jsonl fixture is not a real zip).
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
@@ -238,28 +238,59 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => {
expect(data.max_bytes).toBe(1024 * 1024 * 1024);
});
test.each(["tts", "image", "video"])(
"dataset upload --dry-run 接受媒体 schema %s",
async (schema) => {
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"upload",
"--file",
file,
"--schema",
schema,
"--no-validate",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{ action: string; schema: string }>(stdout);
expect(data.action).toBe("dataset.upload");
expect(data.schema).toBe(schema);
},
);
test.each(["tts", "image"])("dataset upload --dry-run 接受媒体 schema %s", async (schema) => {
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"upload",
"--file",
file,
"--schema",
schema,
"--no-validate",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{ action: string; schema: string }>(stdout);
expect(data.action).toBe("dataset.upload");
expect(data.schema).toBe(schema);
});
test("dataset validate --schema video 拒绝（视频生成入口已隐藏）", async () => {
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"validate",
"--file",
file,
"--schema",
"video",
"--output",
"json",
]);
expect(exitCode, stdout + stderr).not.toBe(0);
expect(`${stdout}\n${stderr}`).toMatch(/--schema video is not supported/);
});
test("dataset upload --schema video 拒绝（视频生成入口已隐藏）", async () => {
const file = join(__dirname, ".dataset-valid.jsonl");
const { stdout, stderr, exitCode } = await runCli([
"dataset",
"upload",
"--file",
file,
"--schema",
"video",
"--no-validate",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stdout + stderr).not.toBe(0);
expect(`${stdout}\n${stderr}`).toMatch(/--schema video is not supported/);
});
});
describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => {
packages/cli/tests/e2e/deploy.e2e.test.ts
-63
View File
@@ -56,69 +56,6 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => {
expect(data.body.capacity).toBe(1);
});
test("deploy create --dry-run 组装视频 LoRA 的 aigc_config", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
"create",
"--model",
"wan2.5-i2v-preview-ft-xxx",
"--name",
"my-video-lora",
"--aigc-prompt",
"a cat surfing",
"--aigc-lora-prompt-default",
"trigger-word",
"--aigc-use-input-prompt",
"true",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{
action: string;
body: {
plan: string;
aigc_config?: {
use_input_prompt?: boolean;
prompt?: string;
lora_prompt_default?: string;
};
};
}>(stdout);
expect(data.action).toBe("deploy.create");
expect(data.body.plan).toBe("lora");
expect(data.body.aigc_config).toEqual({
use_input_prompt: true,
prompt: "a cat surfing",
lora_prompt_default: "trigger-word",
});
});
test("deploy create --aigc-* 仅对 plan=lora 有效（plan=ptu 时报错）", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
"create",
"--model",
"wan2.5-i2v-preview-ft-xxx",
"--name",
"my-video-lora",
"--plan",
"ptu",
"--input-tpm",
"10000",
"--output-tpm",
"1000",
"--aigc-prompt",
"a cat surfing",
"--dry-run",
"--output",
"json",
]);
expect(exitCode, stdout + stderr).not.toBe(0);
expect(`${stdout}\n${stderr}`).toMatch(/--aigc-\* flags are only valid for plan=lora/);
});
test("deploy create --plan mu --deploy-spec --dry-run 透传 deploy_spec", async () => {
const { stdout, stderr, exitCode } = await runCli([
"deploy",
packages/commands/src/commands/dataset/upload.ts
+22 -17
View File
@@ -18,7 +18,7 @@ const UPLOAD_FLAGS = {
file: {
type: "string",
valueHint: "<path>",
description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video)",
description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image)",
required: true,
},
purpose: {
@@ -30,7 +30,7 @@ const UPLOAD_FLAGS = {
type: "string",
valueHint: "<s>",
description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
},
noValidate: {
type: "switch",
@@ -46,7 +46,7 @@ export default defineCommand({
description: "Upload a dataset file (.jsonl or .zip) to Bailian",
auth: "apiKey",
usageArgs:
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image|video>] [--no-validate] [--full-validate]",
"--file <path> [--purpose <name>] [--schema <chatml|dpo|cpt|tts|image>] [--no-validate] [--full-validate]",
flags: UPLOAD_FLAGS,
exampleArgs: [
"--file train.jsonl",
@@ -58,27 +58,32 @@ export default defineCommand({
"--file train.jsonl --no-validate",
],
notes: [
"Supports .jsonl (text) and .zip (audio/image/video archives with a",
"data.jsonl manifest). Six record schemas are recognized: chatml =",
"{messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};",
'cpt = {text:"..."} (continual pre-training, raw text); tts =',
'{wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =',
'{img_path:"..."} (image generation); video = {first_frame_path:"...",',
'video_path:"..."} (video generation). With no --schema, a record',
"carrying wav_fn is validated as TTS, img_path as image, video_path /",
"first_frame_path as video, chosen/rejected as DPO, text (no messages)",
"as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.",
"Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so",
"the purpose tag is persisted (the DashScope-native /api/v1/files drops it).",
"Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl",
"manifest). Five record schemas are recognized: chatml = {messages:[...]}",
'(SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."}',
'(continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav",',
'text:"..."} (audio fine-tuning); image = {img_path:"..."} (image',
"generation). With no --schema, a record carrying wav_fn is validated as",
"TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT,",
"otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the",
"OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is",
"persisted (the DashScope-native /api/v1/files drops it).",
],
async run(ctx) {
const { identity, settings, flags } = ctx;
const filePath = flags.file;
const purpose = flags.purpose || "fine-tune";
const schema = parseDatasetSchemaFlag(flags.schema);
if (schema === "video") {
throw new BailianError(
`--schema video is not supported.`,
ExitCode.USAGE,
`Supported schemas: chatml, dpo, cpt, tts, image.`,
);
}
const format = detectOutputFormat(settings.output);
// Image / video schemas allow larger ZIPs (1 GB vs 300 MB for text).
const isMediaSchema = schema === "image" || schema === "video";
// Image schema allows larger ZIPs (1 GB vs 300 MB for text).
const isMediaSchema = schema === "image";
if (!flags.noValidate) {
const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES;
packages/commands/src/commands/dataset/validate.ts
+16 -10
View File
@@ -36,7 +36,7 @@ const VALIDATE_FLAGS = {
type: "string",
valueHint: "<s>",
description:
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.',
'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.',
},
} satisfies FlagsDef;
@@ -44,7 +44,7 @@ export default defineCommand({
description: "Locally validate a dataset file (.jsonl or .zip) without uploading",
// 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。
auth: "none",
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image|video>]",
usageArgs: "--file <path> [--full-validate] [--schema <chatml|dpo|cpt|tts|image>]",
flags: VALIDATE_FLAGS,
exampleArgs: [
"--file train.jsonl",
@@ -60,19 +60,25 @@ export default defineCommand({
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
'rejected}; cpt = {text:"..."} (continual pre-training, raw text);',
'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);',
'image = {img_path:"..."} (image generation); video =',
'{first_frame_path:"...", video_path:"..."} (video generation). With no',
"--schema, a record carrying wav_fn is validated as TTS, img_path as",
"image, video_path / first_frame_path as video, chosen/rejected as DPO,",
"text (no messages) as CPT, otherwise ChatML. Pass --schema to require a",
"specific shape on every record. ZIP archives (.zip) are validated",
"structurally (data.jsonl present, media references resolve) in addition",
"to per-record content checks. Use --full-validate to JSON.parse every line.",
'image = {img_path:"..."} (image generation). With no --schema, a record',
"carrying wav_fn is validated as TTS, img_path as image, chosen/rejected",
"as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to",
"require a specific shape on every record. ZIP archives (.zip) are",
"validated structurally (data.jsonl present, media references resolve) in",
"addition to per-record content checks. Use --full-validate to JSON.parse",
"every line.",
],
async run(ctx) {
const { settings, flags } = ctx;
const filePath = flags.file;
const schema = parseDatasetSchemaFlag(flags.schema);
if (schema === "video") {
throw new BailianError(
`--schema video is not supported.`,
ExitCode.USAGE,
`Supported schemas: chatml, dpo, cpt, tts, image.`,
);
}
const format = detectOutputFormat(settings.output);
if (settings.dryRun) {
packages/commands/src/commands/deploy/constants.ts
+41
View File
@@ -0,0 +1,41 @@
/**
* Deploy-domain constants — billing plans, billing methods and template
* charge types. Centralised here so no `deploy` command carries a magic
* string for these server-contract values.
*/
/** Billing plan (`--plan` value, matches the server's deployment plan). */
export const DEPLOY_PLAN = {
/** Token-billed; the CLI default. */
LORA: "lora",
/** Token-billed, provisioned throughput. */
PTU: "ptu",
/** Model-unit-billed. */
MU: "mu",
} as const;
export type DeployPlan = (typeof DEPLOY_PLAN)[keyof typeof DEPLOY_PLAN];
/** CLI default plan when `--plan` is omitted. */
export const DEFAULT_DEPLOY_PLAN: DeployPlan = DEPLOY_PLAN.LORA;
/** Billing method (`billing_method`, plan=mu only). */
export const BILLING_METHOD = {
/** Post-paid (currently the only server-supported value). */
POST_PAY: "POST_PAY",
/** Pre-paid. */
PRE_PAY: "PRE_PAY",
} as const;
export type BillingMethod = (typeof BILLING_METHOD)[keyof typeof BILLING_METHOD];
/** Default billing method for plan=mu when `--billing-method` is omitted. */
export const DEFAULT_BILLING_METHOD: BillingMethod = BILLING_METHOD.POST_PAY;
/** Template charge type (`charge_type`) returned by the deployable-models catalog. */
export const CHARGE_TYPE = {
POST_PAID: "post_paid",
PRE_PAID: "pre_paid",
} as const;
export type ChargeType = (typeof CHARGE_TYPE)[keyof typeof CHARGE_TYPE];
packages/commands/src/commands/deploy/create.ts
+3 -52
View File
@@ -2,13 +2,12 @@ import {
defineCommand,
detectOutputFormat,
createDeployment,
BailianError,
ExitCode,
type CreateDeploymentRequest,
type FlagsDef,
} from "bailian-cli-core";
import { emitResult, emitBare } from "bailian-cli-runtime";
import { pickPlanStrategy, STRATEGIES } from "./plans.ts";
import { DEFAULT_DEPLOY_PLAN } from "./constants.ts";
const CREATE_FLAGS = {
model: {
@@ -58,23 +57,6 @@ const CREATE_FLAGS = {
valueHint: "<n>",
description: "PTU max thinking-output tokens/min (optional, some models)",
},
aigcUseInputPrompt: {
type: "boolean",
valueHint: "<bool>",
description:
"Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template)",
},
aigcPrompt: {
type: "string",
valueHint: "<text>",
description:
"Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false",
},
aigcLoraPromptDefault: {
type: "string",
valueHint: "<text>",
description: "Video LoRA (aigc_config): default trigger-word phrase for the LoRA",
},
} satisfies FlagsDef;
/**
@@ -99,7 +81,6 @@ export default defineCommand({
"--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000",
"--model qwen3-8b --name my-qwen3-mu --plan mu",
"--model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2",
'--model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."',
],
notes: [
"Plan defaults to `lora` (Token-billed). Pass --plan to override.",
@@ -112,9 +93,6 @@ export default defineCommand({
"Use `bl deploy models --source base` to inspect available templates.",
"After creation, status starts at PENDING and transitions to RUNNING.",
"Invoke the deployed model with: bl text chat --model <deployed_model>",
"For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass",
"--aigc-prompt / --aigc-lora-prompt-default (and optionally",
"--aigc-use-input-prompt) to set the deployment's aigc_config.",
"WARNING: --model is overloaded across commands and refers to DIFFERENT",
"values. `bl deploy create --model` takes the exported model_name (e.g.",
"`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`",
@@ -124,7 +102,7 @@ export default defineCommand({
"Do not reuse the value across the two commands.",
],
validate: (flags) => {
const plan = flags.plan || "lora";
const plan = flags.plan || DEFAULT_DEPLOY_PLAN;
const strategy = STRATEGIES[plan];
if (!strategy) {
return `Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`;
@@ -135,7 +113,7 @@ export default defineCommand({
const { identity, settings, flags } = ctx;
const model = flags.model;
const name = flags.name;
const plan = flags.plan || "lora";
const plan = flags.plan || DEFAULT_DEPLOY_PLAN;
const format = detectOutputFormat(settings.output);
// Plan-specific behaviour is owned by `plans.ts`. The strategy resolves
@@ -159,33 +137,6 @@ export default defineCommand({
...resolved.body,
};
// AIGC config (fine-tuned Wan video LoRA deployments). Only valid for
// plan=lora — reject early for ptu/mu so the user gets a clear CLI error
// instead of an opaque server-side rejection.
const aigcUseInputPrompt = flags.aigcUseInputPrompt;
const aigcPrompt = flags.aigcPrompt;
const aigcLoraPromptDefault = flags.aigcLoraPromptDefault;
const hasAigcFlags =
aigcUseInputPrompt !== undefined ||
aigcPrompt !== undefined ||
aigcLoraPromptDefault !== undefined;
if (hasAigcFlags && plan !== "lora") {
throw new BailianError(
`--aigc-* flags are only valid for plan=lora (video LoRA deployments). Got plan=${plan}.`,
ExitCode.USAGE,
);
}
if (hasAigcFlags) {
const aigcConfig: Record<string, unknown> = {
use_input_prompt: aigcUseInputPrompt ?? false,
};
if (aigcPrompt !== undefined) aigcConfig.prompt = aigcPrompt;
if (aigcLoraPromptDefault !== undefined) {
aigcConfig.lora_prompt_default = aigcLoraPromptDefault;
}
body.aigc_config = aigcConfig;
}
if (settings.dryRun) {
emitResult({ action: "deploy.create", body }, format);
return;
packages/commands/src/commands/deploy/plans.ts
+11 -9
View File
@@ -14,6 +14,7 @@
* auto-pick / body assembly) into one strategy entry per plan.
*/
import { listDeployableModels, BailianError, ExitCode, type Client } from "bailian-cli-core";
import { DEPLOY_PLAN, BILLING_METHOD, CHARGE_TYPE, DEFAULT_BILLING_METHOD } from "./constants.ts";
/** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */
export interface CreatePlanFlags {
@@ -65,7 +66,7 @@ export interface PlanStrategy {
* the CLI injects `1` as a placeholder.
*/
const loraStrategy: PlanStrategy = {
name: "lora",
name: DEPLOY_PLAN.LORA,
validateFlags() {
return undefined; /* no required flags */
},
@@ -81,7 +82,7 @@ const loraStrategy: PlanStrategy = {
* required.
*/
const ptuStrategy: PlanStrategy = {
name: "ptu",
name: DEPLOY_PLAN.PTU,
validateFlags(flags) {
if (flags.inputTpm === undefined || flags.outputTpm === undefined) {
return "--input-tpm and --output-tpm are required for plan=ptu.";
@@ -115,12 +116,12 @@ const ptuStrategy: PlanStrategy = {
* It is also skipped in dry-run mode to keep `--dry-run` side-effect-free.
*/
const muStrategy: PlanStrategy = {
name: "mu",
name: DEPLOY_PLAN.MU,
validateFlags() {
return undefined; /* every required field has a default — nothing to assert up-front */
},
async resolve(ctx: PlanContext): Promise<PlanResolved> {
const billingMethod = ctx.flags.billingMethod || "POST_PAY";
const billingMethod = ctx.flags.billingMethod || DEFAULT_BILLING_METHOD;
let deploySpec = ctx.flags.deploySpec;
let capacity = ctx.flags.capacity;
@@ -140,11 +141,12 @@ const muStrategy: PlanStrategy = {
});
const payload = resp.output ?? resp.data;
const target = (payload?.models ?? []).find((model) => model.model_name === ctx.model);
const muPlan = target?.plans?.find((plan) => plan.plan === "mu");
const muPlan = target?.plans?.find(({ plan }) => plan === DEPLOY_PLAN.MU);
const templates = muPlan?.templates ?? [];
if (templates.length === 0) throw noTemplateError();
// POST_PAY → post_paid template; fall back to the first available.
const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid";
const wantChargeType =
billingMethod === BILLING_METHOD.POST_PAY ? CHARGE_TYPE.POST_PAID : CHARGE_TYPE.PRE_PAID;
const picked =
templates.find((template) => template.charge_type === wantChargeType) ?? templates[0];
if (!picked?.deploy_spec && !picked?.template_id) throw noTemplateError();
@@ -178,9 +180,9 @@ const muStrategy: PlanStrategy = {
* reject anything outside this table with a clear USAGE error.
*/
export const STRATEGIES: Record<string, PlanStrategy> = {
lora: loraStrategy,
ptu: ptuStrategy,
mu: muStrategy,
[DEPLOY_PLAN.LORA]: loraStrategy,
[DEPLOY_PLAN.PTU]: ptuStrategy,
[DEPLOY_PLAN.MU]: muStrategy,
};
/** Throws USAGE if `plan` is not in the strategy table. */
packages/commands/src/commands/finetune/create.ts
+10
View File
@@ -337,6 +337,16 @@ export default defineCommand({
.map((token) => token.trim())
.find((token) => isLocalPath(token));
const modality: DataModality = firstLocalPath ? await detectModality(firstLocalPath) : "text";
// Video generation model fine-tuning is currently not exposed via the CLI.
// The core profile/schema implementation is retained, but the entry point
// is disabled here so video datasets are not accepted.
if (modality === "video" || modality === "video-kf2v") {
throw new BailianError(
`Video generation model fine-tuning is not supported.`,
ExitCode.USAGE,
`Detected video data in "${firstLocalPath}". Supported data: text, audio, image.`,
);
}
const training = await analyzeDatasetTokens(
settings,
packages/core/src/dataset/inspect.ts
+2 -2
View File
@@ -21,6 +21,7 @@ import { createInterface } from "readline";
import { extname } from "path";
import { BailianError } from "../errors/base.ts";
import { ExitCode } from "../errors/codes.ts";
import { openZipAndFindEntry } from "./validate/zip.ts";
import type { DataModality } from "../finetune/profiles/types.ts";
/**
@@ -159,8 +160,7 @@ function readFirstNonBlankLine(filePath: string): Promise<string | null> {
* `validate/zip.ts` — avoids duplicating yauzl boilerplate.
*/
function readFirstLineFromZipEntry(zipPath: string, entryName: string): Promise<string | null> {
return import("./validate/zip.ts")
.then(({ openZipAndFindEntry }) => openZipAndFindEntry(zipPath, entryName))
return openZipAndFindEntry(zipPath, entryName)
.then(({ entry, zipfile }) => {
return new Promise<string | null>((resolve, reject) => {
zipfile.openReadStream(entry, (streamErr, readStream) => {
packages/core/src/dataset/validate/common.ts
+2 -2
View File
@@ -85,9 +85,9 @@ export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema
if (v === "chatml" || v === "dpo" || v === "cpt" || v === "tts" || v === "image" || v === "video")
return v;
throw new BailianError(
`Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image, video.`,
`Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image.`,
ExitCode.USAGE,
`Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, first_frame_path/video_path → video, else ChatML).`,
`Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, else ChatML).`,
);
}
packages/core/src/dataset/validate/zip.ts
+40 -48
View File
@@ -16,11 +16,13 @@
* `"tts"` for audio). The profile layer decides which schema to use based on
* the detected modality — this validator is schema-agnostic.
*/
import { createWriteStream, mkdirSync, rmSync } from "fs";
import { createReadStream, createWriteStream, mkdirSync, rmSync } from "fs";
import { createInterface } from "readline";
import { tmpdir } from "os";
import { join } from "path";
import { pipeline } from "stream/promises";
import { randomBytes } from "crypto";
import * as yauzl from "yauzl";
import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts";
import { makeIssue } from "./common.ts";
import { jsonlValidator } from "./jsonl.ts";
@@ -39,32 +41,28 @@ import { IMAGE_EXTENSIONS } from "./schemas/image.ts";
export function openZipAndFindEntry(
zipPath: string,
targetName: string,
): Promise<{ entry: import("yauzl").Entry; zipfile: import("yauzl").ZipFile }> {
): Promise<{ entry: yauzl.Entry; zipfile: yauzl.ZipFile }> {
return new Promise((resolve, reject) => {
import("yauzl")
.then((yauzl) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
zipfile.readEntry();
zipfile.on("entry", (entry) => {
const name = entry.fileName.replace(/\\/g, "/");
if (name === targetName || name.endsWith(`/${targetName}`)) {
resolve({ entry, zipfile });
} else {
zipfile.readEntry();
zipfile.on("entry", (entry) => {
const name = entry.fileName.replace(/\\/g, "/");
if (name === targetName || name.endsWith(`/${targetName}`)) {
resolve({ entry, zipfile });
} else {
zipfile.readEntry();
}
});
zipfile.on("end", () => {
zipfile.close();
reject(new Error(`Entry "${targetName}" not found in ZIP`));
});
zipfile.on("error", reject);
});
})
.catch(reject);
}
});
zipfile.on("end", () => {
zipfile.close();
reject(new Error(`Entry "${targetName}" not found in ZIP`));
});
zipfile.on("error", reject);
});
});
}
@@ -74,27 +72,23 @@ export function openZipAndFindEntry(
*/
function collectZipEntries(zipPath: string): Promise<string[]> {
return new Promise((resolve, reject) => {
import("yauzl")
.then((yauzl) => {
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
const entries: string[] = [];
zipfile.readEntry();
zipfile.on("entry", (entry) => {
entries.push(entry.fileName.replace(/\\/g, "/"));
zipfile.readEntry();
});
zipfile.on("end", () => {
zipfile.close();
resolve(entries);
});
zipfile.on("error", reject);
});
})
.catch(reject);
yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
if (err || !zipfile) {
reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`));
return;
}
const entries: string[] = [];
zipfile.readEntry();
zipfile.on("entry", (entry) => {
entries.push(entry.fileName.replace(/\\/g, "/"));
zipfile.readEntry();
});
zipfile.on("end", () => {
zipfile.close();
resolve(entries);
});
zipfile.on("error", reject);
});
});
}
@@ -140,8 +134,6 @@ async function collectMediaRefs(
jsonlPath: string,
maxLines = 100,
): Promise<{ refs: string[]; totalLines: number }> {
const { createReadStream } = await import("fs");
const { createInterface } = await import("readline");
const stream = createReadStream(jsonlPath, { encoding: "utf8" });
const rl = createInterface({ input: stream, crlfDelay: Infinity });
const refs: string[] = [];
packages/core/src/finetune/profiles/sft-lora.ts
+1 -1
View File
@@ -199,7 +199,7 @@ export const sftLoraProfile: TrainingProfile = {
const wan25 = isWan25(flags.model as string | undefined);
const hp: Record<string, unknown> = {
...VIDEO_HYPER_PARAMS_BASE,
batch_size: wan25 ? 2 : 4,
batch_size: 4,
max_pixels: wan25 ? 36864 : 262144,
};
// Optional overrides (no clamping — video batch_size is intentionally small).
skills/bailian-cli/reference/dataset.md
+41 -44
View File
@@ -107,38 +107,36 @@ bl dataset list --output json
### `bl dataset upload`
| Field | Value |
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `dataset upload` |
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>] [--no-validate] [--full-validate]` |
| Field | Value |
| --------------- | -------------------------------------------------------------------------------------------------------------------------------- |
| **Name** | `dataset upload` |
| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian |
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo\|cpt\|tts\|image>] [--no-validate] [--full-validate]` |
#### Flags
| Flag | Type | Required | Description |
| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video) |
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
| Flag | Type | Required | Description |
| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) |
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
#### Notes
- Supports .jsonl (text) and .zip (audio/image/video archives with a
- data.jsonl manifest). Six record schemas are recognized: chatml =
- {messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};
- cpt = {text:"..."} (continual pre-training, raw text); tts =
- {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =
- {img_path:"..."} (image generation); video = {first_frame_path:"...",
- video_path:"..."} (video generation). With no --schema, a record
- carrying wav_fn is validated as TTS, img_path as image, video_path /
- first_frame_path as video, chosen/rejected as DPO, text (no messages)
- as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.
- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so
- the purpose tag is persisted (the DashScope-native /api/v1/files drops it).
- Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl
- manifest). Five record schemas are recognized: chatml = {messages:[...]}
- (SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."}
- (continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav",
- text:"..."} (audio fine-tuning); image = {img_path:"..."} (image
- generation). With no --schema, a record carrying wav_fn is validated as
- TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT,
- otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the
- OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is
- persisted (the DashScope-native /api/v1/files drops it).
#### Examples
@@ -172,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate
### `bl dataset validate`
| Field | Value |
| --------------- | ------------------------------------------------------------------------------------------------------ |
| **Name** | `dataset validate` |
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image\|video>]` |
| Field | Value |
| --------------- | ----------------------------------------------------------------------------------------------- |
| **Name** | `dataset validate` |
| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading |
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo\|cpt\|tts\|image>]` |
#### Flags
| Flag | Type | Required | Description |
| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. |
| Flag | Type | Required | Description |
| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--file <path>` | string | yes | Local dataset file (.jsonl or .zip) |
| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) |
| `--schema <s>` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. |
#### Notes
@@ -193,14 +191,13 @@ bl dataset upload --file train.jsonl --no-validate
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
- rejected}; cpt = {text:"..."} (continual pre-training, raw text);
- tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);
- image = {img_path:"..."} (image generation); video =
- {first_frame_path:"...", video_path:"..."} (video generation). With no
- --schema, a record carrying wav_fn is validated as TTS, img_path as
- image, video_path / first_frame_path as video, chosen/rejected as DPO,
- text (no messages) as CPT, otherwise ChatML. Pass --schema to require a
- specific shape on every record. ZIP archives (.zip) are validated
- structurally (data.jsonl present, media references resolve) in addition
- to per-record content checks. Use --full-validate to JSON.parse every line.
- image = {img_path:"..."} (image generation). With no --schema, a record
- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected
- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to
- require a specific shape on every record. ZIP archives (.zip) are
- validated structurally (data.jsonl present, media references resolve) in
- addition to per-record content checks. Use --full-validate to JSON.parse
- every line.
#### Examples
skills/bailian-cli/reference/deploy.md
+13 -23
View File
@@ -29,22 +29,19 @@ Index: [index.md](index.md)
#### Flags
| Flag | Type | Required | Description |
| ----------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------ |
| `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) |
| `--name <display_name>` | string | yes | Console display name for the deployment (required) |
| `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu |
| `--deploy-spec <id>` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) |
| `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) |
| `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) |
| `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) |
| `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) |
| `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) |
| `--aigc-use-input-prompt <bool>` | boolean | no | Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template) |
| `--aigc-prompt <text>` | string | no | Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false |
| `--aigc-lora-prompt-default <text>` | string | no | Video LoRA (aigc_config): default trigger-word phrase for the LoRA |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
| Flag | Type | Required | Description |
| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- |
| `--model <name>` | string | yes | Model name (catalog model or fine-tuned output) (required) |
| `--name <display_name>` | string | yes | Console display name for the deployment (required) |
| `--plan <plan>` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu |
| `--deploy-spec <id>` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) |
| `--capacity <n>` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) |
| `--billing-method <m>` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) |
| `--input-tpm <n>` | number | no | PTU max input tokens/min (required for plan=ptu) |
| `--output-tpm <n>` | number | no | PTU max output tokens/min (required for plan=ptu) |
| `--thinking-output-tpm <n>` | number | no | PTU max thinking-output tokens/min (optional, some models) |
| `--api-key <key>` | string | no | API key |
| `--base-url <url>` | string | no | API base URL |
#### Notes
@@ -58,9 +55,6 @@ Index: [index.md](index.md)
- Use `bl deploy models --source base` to inspect available templates.
- After creation, status starts at PENDING and transitions to RUNNING.
- Invoke the deployed model with: bl text chat --model <deployed_model>
- For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass
- --aigc-prompt / --aigc-lora-prompt-default (and optionally
- --aigc-use-input-prompt) to set the deployment's aigc_config.
- WARNING: --model is overloaded across commands and refers to DIFFERENT
- values. `bl deploy create --model` takes the exported model_name (e.g.
- `qwen3-8b-ft-...`), but the create response also returns a `deployed_model`
@@ -87,10 +81,6 @@ bl deploy create --model qwen3-8b --name my-qwen3-mu --plan mu
bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2
```
```bash
bl deploy create --model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."
```
### `bl deploy delete`
| Field | Value |