diff --git a/packages/cli/tests/e2e/dataset.e2e.test.ts b/packages/cli/tests/e2e/dataset.e2e.test.ts index 50acb54..2118ff8 100644 --- a/packages/cli/tests/e2e/dataset.e2e.test.ts +++ b/packages/cli/tests/e2e/dataset.e2e.test.ts @@ -216,7 +216,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => { }); test("dataset upload --schema image --no-validate --dry-run 采用 1GB 媒体上限", async () => { - // image/video schemas raise the upload cap to 1 GiB (vs 300 MB for text). + // image schema raises the upload cap to 1 GiB (vs 300 MB for text). // --no-validate keeps this offline (the jsonl fixture is not a real zip). const file = join(__dirname, ".dataset-valid.jsonl"); const { stdout, stderr, exitCode } = await runCli([ @@ -238,28 +238,59 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (offline)", () => { expect(data.max_bytes).toBe(1024 * 1024 * 1024); }); - test.each(["tts", "image", "video"])( - "dataset upload --dry-run 接受媒体 schema %s", - async (schema) => { - const file = join(__dirname, ".dataset-valid.jsonl"); - const { stdout, stderr, exitCode } = await runCli([ - "dataset", - "upload", - "--file", - file, - "--schema", - schema, - "--no-validate", - "--dry-run", - "--output", - "json", - ]); - expect(exitCode, stderr).toBe(0); - const data = parseStdoutJson<{ action: string; schema: string }>(stdout); - expect(data.action).toBe("dataset.upload"); - expect(data.schema).toBe(schema); - }, - ); + test.each(["tts", "image"])("dataset upload --dry-run 接受媒体 schema %s", async (schema) => { + const file = join(__dirname, ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "upload", + "--file", + file, + "--schema", + schema, + "--no-validate", + "--dry-run", + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ action: string; schema: string }>(stdout); + expect(data.action).toBe("dataset.upload"); + expect(data.schema).toBe(schema); + }); + + test("dataset validate --schema video 拒绝（视频生成入口已隐藏）", async () => { + const file = join(__dirname, ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "validate", + "--file", + file, + "--schema", + "video", + "--output", + "json", + ]); + expect(exitCode, stdout + stderr).not.toBe(0); + expect(`${stdout}\n${stderr}`).toMatch(/--schema video is not supported/); + }); + + test("dataset upload --schema video 拒绝（视频生成入口已隐藏）", async () => { + const file = join(__dirname, ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "upload", + "--file", + file, + "--schema", + "video", + "--no-validate", + "--dry-run", + "--output", + "json", + ]); + expect(exitCode, stdout + stderr).not.toBe(0); + expect(`${stdout}\n${stderr}`).toMatch(/--schema video is not supported/); + }); }); describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => { diff --git a/packages/cli/tests/e2e/deploy.e2e.test.ts b/packages/cli/tests/e2e/deploy.e2e.test.ts index 7c3a47f..d40ac2c 100644 --- a/packages/cli/tests/e2e/deploy.e2e.test.ts +++ b/packages/cli/tests/e2e/deploy.e2e.test.ts @@ -56,69 +56,6 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => { expect(data.body.capacity).toBe(1); }); - test("deploy create --dry-run 组装视频 LoRA 的 aigc_config", async () => { - const { stdout, stderr, exitCode } = await runCli([ - "deploy", - "create", - "--model", - "wan2.5-i2v-preview-ft-xxx", - "--name", - "my-video-lora", - "--aigc-prompt", - "a cat surfing", - "--aigc-lora-prompt-default", - "trigger-word", - "--aigc-use-input-prompt", - "true", - "--dry-run", - "--output", - "json", - ]); - expect(exitCode, stderr).toBe(0); - const data = parseStdoutJson<{ - action: string; - body: { - plan: string; - aigc_config?: { - use_input_prompt?: boolean; - prompt?: string; - lora_prompt_default?: string; - }; - }; - }>(stdout); - expect(data.action).toBe("deploy.create"); - expect(data.body.plan).toBe("lora"); - expect(data.body.aigc_config).toEqual({ - use_input_prompt: true, - prompt: "a cat surfing", - lora_prompt_default: "trigger-word", - }); - }); - - test("deploy create --aigc-* 仅对 plan=lora 有效（plan=ptu 时报错）", async () => { - const { stdout, stderr, exitCode } = await runCli([ - "deploy", - "create", - "--model", - "wan2.5-i2v-preview-ft-xxx", - "--name", - "my-video-lora", - "--plan", - "ptu", - "--input-tpm", - "10000", - "--output-tpm", - "1000", - "--aigc-prompt", - "a cat surfing", - "--dry-run", - "--output", - "json", - ]); - expect(exitCode, stdout + stderr).not.toBe(0); - expect(`${stdout}\n${stderr}`).toMatch(/--aigc-\* flags are only valid for plan=lora/); - }); - test("deploy create --plan mu --deploy-spec --dry-run 透传 deploy_spec", async () => { const { stdout, stderr, exitCode } = await runCli([ "deploy", diff --git a/packages/commands/src/commands/dataset/upload.ts b/packages/commands/src/commands/dataset/upload.ts index ed0a7e1..f7d805b 100644 --- a/packages/commands/src/commands/dataset/upload.ts +++ b/packages/commands/src/commands/dataset/upload.ts @@ -18,7 +18,7 @@ const UPLOAD_FLAGS = { file: { type: "string", valueHint: "", - description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video)", + description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image)", required: true, }, purpose: { @@ -30,7 +30,7 @@ const UPLOAD_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', }, noValidate: { type: "switch", @@ -46,7 +46,7 @@ export default defineCommand({ description: "Upload a dataset file (.jsonl or .zip) to Bailian", auth: "apiKey", usageArgs: - "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", + "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", flags: UPLOAD_FLAGS, exampleArgs: [ "--file train.jsonl", @@ -58,27 +58,32 @@ export default defineCommand({ "--file train.jsonl --no-validate", ], notes: [ - "Supports .jsonl (text) and .zip (audio/image/video archives with a", - "data.jsonl manifest). Six record schemas are recognized: chatml =", - "{messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected};", - 'cpt = {text:"..."} (continual pre-training, raw text); tts =', - '{wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image =', - '{img_path:"..."} (image generation); video = {first_frame_path:"...",', - 'video_path:"..."} (video generation). With no --schema, a record', - "carrying wav_fn is validated as TTS, img_path as image, video_path /", - "first_frame_path as video, chosen/rejected as DPO, text (no messages)", - "as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video.", - "Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so", - "the purpose tag is persisted (the DashScope-native /api/v1/files drops it).", + "Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl", + "manifest). Five record schemas are recognized: chatml = {messages:[...]}", + '(SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."}', + '(continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav",', + 'text:"..."} (audio fine-tuning); image = {img_path:"..."} (image', + "generation). With no --schema, a record carrying wav_fn is validated as", + "TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT,", + "otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the", + "OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is", + "persisted (the DashScope-native /api/v1/files drops it).", ], async run(ctx) { const { identity, settings, flags } = ctx; const filePath = flags.file; const purpose = flags.purpose || "fine-tune"; const schema = parseDatasetSchemaFlag(flags.schema); + if (schema === "video") { + throw new BailianError( + `--schema video is not supported.`, + ExitCode.USAGE, + `Supported schemas: chatml, dpo, cpt, tts, image.`, + ); + } const format = detectOutputFormat(settings.output); - // Image / video schemas allow larger ZIPs (1 GB vs 300 MB for text). - const isMediaSchema = schema === "image" || schema === "video"; + // Image schema allows larger ZIPs (1 GB vs 300 MB for text). + const isMediaSchema = schema === "image"; if (!flags.noValidate) { const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES; diff --git a/packages/commands/src/commands/dataset/validate.ts b/packages/commands/src/commands/dataset/validate.ts index f9c8cb0..8bf0d29 100644 --- a/packages/commands/src/commands/dataset/validate.ts +++ b/packages/commands/src/commands/dataset/validate.ts @@ -36,7 +36,7 @@ const VALIDATE_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', }, } satisfies FlagsDef; @@ -44,7 +44,7 @@ export default defineCommand({ description: "Locally validate a dataset file (.jsonl or .zip) without uploading", // 纯本地校验，不触网、不需 API key（与 `pipeline validate` 一致）。 auth: "none", - usageArgs: "--file [--full-validate] [--schema ]", + usageArgs: "--file [--full-validate] [--schema ]", flags: VALIDATE_FLAGS, exampleArgs: [ "--file train.jsonl", @@ -60,19 +60,25 @@ export default defineCommand({ "Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,", 'rejected}; cpt = {text:"..."} (continual pre-training, raw text);', 'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);', - 'image = {img_path:"..."} (image generation); video =', - '{first_frame_path:"...", video_path:"..."} (video generation). With no', - "--schema, a record carrying wav_fn is validated as TTS, img_path as", - "image, video_path / first_frame_path as video, chosen/rejected as DPO,", - "text (no messages) as CPT, otherwise ChatML. Pass --schema to require a", - "specific shape on every record. ZIP archives (.zip) are validated", - "structurally (data.jsonl present, media references resolve) in addition", - "to per-record content checks. Use --full-validate to JSON.parse every line.", + 'image = {img_path:"..."} (image generation). With no --schema, a record', + "carrying wav_fn is validated as TTS, img_path as image, chosen/rejected", + "as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to", + "require a specific shape on every record. ZIP archives (.zip) are", + "validated structurally (data.jsonl present, media references resolve) in", + "addition to per-record content checks. Use --full-validate to JSON.parse", + "every line.", ], async run(ctx) { const { settings, flags } = ctx; const filePath = flags.file; const schema = parseDatasetSchemaFlag(flags.schema); + if (schema === "video") { + throw new BailianError( + `--schema video is not supported.`, + ExitCode.USAGE, + `Supported schemas: chatml, dpo, cpt, tts, image.`, + ); + } const format = detectOutputFormat(settings.output); if (settings.dryRun) { diff --git a/packages/commands/src/commands/deploy/constants.ts b/packages/commands/src/commands/deploy/constants.ts new file mode 100644 index 0000000..748645f --- /dev/null +++ b/packages/commands/src/commands/deploy/constants.ts @@ -0,0 +1,41 @@ +/** + * Deploy-domain constants — billing plans, billing methods and template + * charge types. Centralised here so no `deploy` command carries a magic + * string for these server-contract values. + */ + +/** Billing plan (`--plan` value, matches the server's deployment plan). */ +export const DEPLOY_PLAN = { + /** Token-billed; the CLI default. */ + LORA: "lora", + /** Token-billed, provisioned throughput. */ + PTU: "ptu", + /** Model-unit-billed. */ + MU: "mu", +} as const; + +export type DeployPlan = (typeof DEPLOY_PLAN)[keyof typeof DEPLOY_PLAN]; + +/** CLI default plan when `--plan` is omitted. */ +export const DEFAULT_DEPLOY_PLAN: DeployPlan = DEPLOY_PLAN.LORA; + +/** Billing method (`billing_method`, plan=mu only). */ +export const BILLING_METHOD = { + /** Post-paid (currently the only server-supported value). */ + POST_PAY: "POST_PAY", + /** Pre-paid. */ + PRE_PAY: "PRE_PAY", +} as const; + +export type BillingMethod = (typeof BILLING_METHOD)[keyof typeof BILLING_METHOD]; + +/** Default billing method for plan=mu when `--billing-method` is omitted. */ +export const DEFAULT_BILLING_METHOD: BillingMethod = BILLING_METHOD.POST_PAY; + +/** Template charge type (`charge_type`) returned by the deployable-models catalog. */ +export const CHARGE_TYPE = { + POST_PAID: "post_paid", + PRE_PAID: "pre_paid", +} as const; + +export type ChargeType = (typeof CHARGE_TYPE)[keyof typeof CHARGE_TYPE]; diff --git a/packages/commands/src/commands/deploy/create.ts b/packages/commands/src/commands/deploy/create.ts index d9a63e8..6f4a2dd 100644 --- a/packages/commands/src/commands/deploy/create.ts +++ b/packages/commands/src/commands/deploy/create.ts @@ -2,13 +2,12 @@ import { defineCommand, detectOutputFormat, createDeployment, - BailianError, - ExitCode, type CreateDeploymentRequest, type FlagsDef, } from "bailian-cli-core"; import { emitResult, emitBare } from "bailian-cli-runtime"; import { pickPlanStrategy, STRATEGIES } from "./plans.ts"; +import { DEFAULT_DEPLOY_PLAN } from "./constants.ts"; const CREATE_FLAGS = { model: { @@ -58,23 +57,6 @@ const CREATE_FLAGS = { valueHint: "", description: "PTU max thinking-output tokens/min (optional, some models)", }, - aigcUseInputPrompt: { - type: "boolean", - valueHint: "", - description: - "Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template)", - }, - aigcPrompt: { - type: "string", - valueHint: "", - description: - "Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false", - }, - aigcLoraPromptDefault: { - type: "string", - valueHint: "", - description: "Video LoRA (aigc_config): default trigger-word phrase for the LoRA", - }, } satisfies FlagsDef; /** @@ -99,7 +81,6 @@ export default defineCommand({ "--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000", "--model qwen3-8b --name my-qwen3-mu --plan mu", "--model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2", - '--model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..."', ], notes: [ "Plan defaults to `lora` (Token-billed). Pass --plan to override.", @@ -112,9 +93,6 @@ export default defineCommand({ "Use `bl deploy models --source base` to inspect available templates.", "After creation, status starts at PENDING and transitions to RUNNING.", "Invoke the deployed model with: bl text chat --model ", - "For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass", - "--aigc-prompt / --aigc-lora-prompt-default (and optionally", - "--aigc-use-input-prompt) to set the deployment's aigc_config.", "WARNING: --model is overloaded across commands and refers to DIFFERENT", "values. `bl deploy create --model` takes the exported model_name (e.g.", "`qwen3-8b-ft-...`), but the create response also returns a `deployed_model`", @@ -124,7 +102,7 @@ export default defineCommand({ "Do not reuse the value across the two commands.", ], validate: (flags) => { - const plan = flags.plan || "lora"; + const plan = flags.plan || DEFAULT_DEPLOY_PLAN; const strategy = STRATEGIES[plan]; if (!strategy) { return `Unsupported plan "${plan}". Supported plans: ${Object.keys(STRATEGIES).join(", ")}.`; @@ -135,7 +113,7 @@ export default defineCommand({ const { identity, settings, flags } = ctx; const model = flags.model; const name = flags.name; - const plan = flags.plan || "lora"; + const plan = flags.plan || DEFAULT_DEPLOY_PLAN; const format = detectOutputFormat(settings.output); // Plan-specific behaviour is owned by `plans.ts`. The strategy resolves @@ -159,33 +137,6 @@ export default defineCommand({ ...resolved.body, }; - // AIGC config (fine-tuned Wan video LoRA deployments). Only valid for - // plan=lora — reject early for ptu/mu so the user gets a clear CLI error - // instead of an opaque server-side rejection. - const aigcUseInputPrompt = flags.aigcUseInputPrompt; - const aigcPrompt = flags.aigcPrompt; - const aigcLoraPromptDefault = flags.aigcLoraPromptDefault; - const hasAigcFlags = - aigcUseInputPrompt !== undefined || - aigcPrompt !== undefined || - aigcLoraPromptDefault !== undefined; - if (hasAigcFlags && plan !== "lora") { - throw new BailianError( - `--aigc-* flags are only valid for plan=lora (video LoRA deployments). Got plan=${plan}.`, - ExitCode.USAGE, - ); - } - if (hasAigcFlags) { - const aigcConfig: Record = { - use_input_prompt: aigcUseInputPrompt ?? false, - }; - if (aigcPrompt !== undefined) aigcConfig.prompt = aigcPrompt; - if (aigcLoraPromptDefault !== undefined) { - aigcConfig.lora_prompt_default = aigcLoraPromptDefault; - } - body.aigc_config = aigcConfig; - } - if (settings.dryRun) { emitResult({ action: "deploy.create", body }, format); return; diff --git a/packages/commands/src/commands/deploy/plans.ts b/packages/commands/src/commands/deploy/plans.ts index 93f8c9d..bde40a4 100644 --- a/packages/commands/src/commands/deploy/plans.ts +++ b/packages/commands/src/commands/deploy/plans.ts @@ -14,6 +14,7 @@ * auto-pick / body assembly) into one strategy entry per plan. */ import { listDeployableModels, BailianError, ExitCode, type Client } from "bailian-cli-core"; +import { DEPLOY_PLAN, BILLING_METHOD, CHARGE_TYPE, DEFAULT_BILLING_METHOD } from "./constants.ts"; /** Plan-relevant subset of `deploy create` flags (parsed flags satisfy this shape). */ export interface CreatePlanFlags { @@ -65,7 +66,7 @@ export interface PlanStrategy { * the CLI injects `1` as a placeholder. */ const loraStrategy: PlanStrategy = { - name: "lora", + name: DEPLOY_PLAN.LORA, validateFlags() { return undefined; /* no required flags */ }, @@ -81,7 +82,7 @@ const loraStrategy: PlanStrategy = { * required. */ const ptuStrategy: PlanStrategy = { - name: "ptu", + name: DEPLOY_PLAN.PTU, validateFlags(flags) { if (flags.inputTpm === undefined || flags.outputTpm === undefined) { return "--input-tpm and --output-tpm are required for plan=ptu."; @@ -115,12 +116,12 @@ const ptuStrategy: PlanStrategy = { * It is also skipped in dry-run mode to keep `--dry-run` side-effect-free. */ const muStrategy: PlanStrategy = { - name: "mu", + name: DEPLOY_PLAN.MU, validateFlags() { return undefined; /* every required field has a default — nothing to assert up-front */ }, async resolve(ctx: PlanContext): Promise { - const billingMethod = ctx.flags.billingMethod || "POST_PAY"; + const billingMethod = ctx.flags.billingMethod || DEFAULT_BILLING_METHOD; let deploySpec = ctx.flags.deploySpec; let capacity = ctx.flags.capacity; @@ -140,11 +141,12 @@ const muStrategy: PlanStrategy = { }); const payload = resp.output ?? resp.data; const target = (payload?.models ?? []).find((model) => model.model_name === ctx.model); - const muPlan = target?.plans?.find((plan) => plan.plan === "mu"); + const muPlan = target?.plans?.find(({ plan }) => plan === DEPLOY_PLAN.MU); const templates = muPlan?.templates ?? []; if (templates.length === 0) throw noTemplateError(); // POST_PAY → post_paid template; fall back to the first available. - const wantChargeType = billingMethod === "POST_PAY" ? "post_paid" : "pre_paid"; + const wantChargeType = + billingMethod === BILLING_METHOD.POST_PAY ? CHARGE_TYPE.POST_PAID : CHARGE_TYPE.PRE_PAID; const picked = templates.find((template) => template.charge_type === wantChargeType) ?? templates[0]; if (!picked?.deploy_spec && !picked?.template_id) throw noTemplateError(); @@ -178,9 +180,9 @@ const muStrategy: PlanStrategy = { * reject anything outside this table with a clear USAGE error. */ export const STRATEGIES: Record = { - lora: loraStrategy, - ptu: ptuStrategy, - mu: muStrategy, + [DEPLOY_PLAN.LORA]: loraStrategy, + [DEPLOY_PLAN.PTU]: ptuStrategy, + [DEPLOY_PLAN.MU]: muStrategy, }; /** Throws USAGE if `plan` is not in the strategy table. */ diff --git a/packages/commands/src/commands/finetune/create.ts b/packages/commands/src/commands/finetune/create.ts index 81d5ad1..58907eb 100644 --- a/packages/commands/src/commands/finetune/create.ts +++ b/packages/commands/src/commands/finetune/create.ts @@ -337,6 +337,16 @@ export default defineCommand({ .map((token) => token.trim()) .find((token) => isLocalPath(token)); const modality: DataModality = firstLocalPath ? await detectModality(firstLocalPath) : "text"; + // Video generation model fine-tuning is currently not exposed via the CLI. + // The core profile/schema implementation is retained, but the entry point + // is disabled here so video datasets are not accepted. + if (modality === "video" || modality === "video-kf2v") { + throw new BailianError( + `Video generation model fine-tuning is not supported.`, + ExitCode.USAGE, + `Detected video data in "${firstLocalPath}". Supported data: text, audio, image.`, + ); + } const training = await analyzeDatasetTokens( settings, diff --git a/packages/core/src/dataset/inspect.ts b/packages/core/src/dataset/inspect.ts index 65c43da..9a556ae 100644 --- a/packages/core/src/dataset/inspect.ts +++ b/packages/core/src/dataset/inspect.ts @@ -21,6 +21,7 @@ import { createInterface } from "readline"; import { extname } from "path"; import { BailianError } from "../errors/base.ts"; import { ExitCode } from "../errors/codes.ts"; +import { openZipAndFindEntry } from "./validate/zip.ts"; import type { DataModality } from "../finetune/profiles/types.ts"; /** @@ -159,8 +160,7 @@ function readFirstNonBlankLine(filePath: string): Promise { * `validate/zip.ts` — avoids duplicating yauzl boilerplate. */ function readFirstLineFromZipEntry(zipPath: string, entryName: string): Promise { - return import("./validate/zip.ts") - .then(({ openZipAndFindEntry }) => openZipAndFindEntry(zipPath, entryName)) + return openZipAndFindEntry(zipPath, entryName) .then(({ entry, zipfile }) => { return new Promise((resolve, reject) => { zipfile.openReadStream(entry, (streamErr, readStream) => { diff --git a/packages/core/src/dataset/validate/common.ts b/packages/core/src/dataset/validate/common.ts index 36b6d30..17bdbad 100644 --- a/packages/core/src/dataset/validate/common.ts +++ b/packages/core/src/dataset/validate/common.ts @@ -85,9 +85,9 @@ export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema if (v === "chatml" || v === "dpo" || v === "cpt" || v === "tts" || v === "image" || v === "video") return v; throw new BailianError( - `Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image, video.`, + `Unsupported --schema "${value}". Supported: chatml, dpo, cpt, tts, image.`, ExitCode.USAGE, - `Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, first_frame_path/video_path → video, else ChatML).`, + `Omit --schema to auto-detect per record (chosen/rejected → DPO, text → CPT, wav_fn → TTS, img_path → image, else ChatML).`, ); } diff --git a/packages/core/src/dataset/validate/zip.ts b/packages/core/src/dataset/validate/zip.ts index 5dbd655..4309133 100644 --- a/packages/core/src/dataset/validate/zip.ts +++ b/packages/core/src/dataset/validate/zip.ts @@ -16,11 +16,13 @@ * `"tts"` for audio). The profile layer decides which schema to use based on * the detected modality — this validator is schema-agnostic. */ -import { createWriteStream, mkdirSync, rmSync } from "fs"; +import { createReadStream, createWriteStream, mkdirSync, rmSync } from "fs"; +import { createInterface } from "readline"; import { tmpdir } from "os"; import { join } from "path"; import { pipeline } from "stream/promises"; import { randomBytes } from "crypto"; +import * as yauzl from "yauzl"; import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts"; import { makeIssue } from "./common.ts"; import { jsonlValidator } from "./jsonl.ts"; @@ -39,32 +41,28 @@ import { IMAGE_EXTENSIONS } from "./schemas/image.ts"; export function openZipAndFindEntry( zipPath: string, targetName: string, -): Promise<{ entry: import("yauzl").Entry; zipfile: import("yauzl").ZipFile }> { +): Promise<{ entry: yauzl.Entry; zipfile: yauzl.ZipFile }> { return new Promise((resolve, reject) => { - import("yauzl") - .then((yauzl) => { - yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => { - if (err || !zipfile) { - reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`)); - return; - } + yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => { + if (err || !zipfile) { + reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`)); + return; + } + zipfile.readEntry(); + zipfile.on("entry", (entry) => { + const name = entry.fileName.replace(/\\/g, "/"); + if (name === targetName || name.endsWith(`/${targetName}`)) { + resolve({ entry, zipfile }); + } else { zipfile.readEntry(); - zipfile.on("entry", (entry) => { - const name = entry.fileName.replace(/\\/g, "/"); - if (name === targetName || name.endsWith(`/${targetName}`)) { - resolve({ entry, zipfile }); - } else { - zipfile.readEntry(); - } - }); - zipfile.on("end", () => { - zipfile.close(); - reject(new Error(`Entry "${targetName}" not found in ZIP`)); - }); - zipfile.on("error", reject); - }); - }) - .catch(reject); + } + }); + zipfile.on("end", () => { + zipfile.close(); + reject(new Error(`Entry "${targetName}" not found in ZIP`)); + }); + zipfile.on("error", reject); + }); }); } @@ -74,27 +72,23 @@ export function openZipAndFindEntry( */ function collectZipEntries(zipPath: string): Promise { return new Promise((resolve, reject) => { - import("yauzl") - .then((yauzl) => { - yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => { - if (err || !zipfile) { - reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`)); - return; - } - const entries: string[] = []; - zipfile.readEntry(); - zipfile.on("entry", (entry) => { - entries.push(entry.fileName.replace(/\\/g, "/")); - zipfile.readEntry(); - }); - zipfile.on("end", () => { - zipfile.close(); - resolve(entries); - }); - zipfile.on("error", reject); - }); - }) - .catch(reject); + yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => { + if (err || !zipfile) { + reject(new Error(`Failed to open ZIP: ${err?.message ?? "unknown error"}`)); + return; + } + const entries: string[] = []; + zipfile.readEntry(); + zipfile.on("entry", (entry) => { + entries.push(entry.fileName.replace(/\\/g, "/")); + zipfile.readEntry(); + }); + zipfile.on("end", () => { + zipfile.close(); + resolve(entries); + }); + zipfile.on("error", reject); + }); }); } @@ -140,8 +134,6 @@ async function collectMediaRefs( jsonlPath: string, maxLines = 100, ): Promise<{ refs: string[]; totalLines: number }> { - const { createReadStream } = await import("fs"); - const { createInterface } = await import("readline"); const stream = createReadStream(jsonlPath, { encoding: "utf8" }); const rl = createInterface({ input: stream, crlfDelay: Infinity }); const refs: string[] = []; diff --git a/packages/core/src/finetune/profiles/sft-lora.ts b/packages/core/src/finetune/profiles/sft-lora.ts index 2319d9a..d442dc5 100644 --- a/packages/core/src/finetune/profiles/sft-lora.ts +++ b/packages/core/src/finetune/profiles/sft-lora.ts @@ -199,7 +199,7 @@ export const sftLoraProfile: TrainingProfile = { const wan25 = isWan25(flags.model as string | undefined); const hp: Record = { ...VIDEO_HYPER_PARAMS_BASE, - batch_size: wan25 ? 2 : 4, + batch_size: 4, max_pixels: wan25 ? 36864 : 262144, }; // Optional overrides (no clamping — video batch_size is intentionally small). diff --git a/skills/bailian-cli/reference/dataset.md b/skills/bailian-cli/reference/dataset.md index 365d84d..523a421 100644 --- a/skills/bailian-cli/reference/dataset.md +++ b/skills/bailian-cli/reference/dataset.md @@ -107,38 +107,36 @@ bl dataset list --output json ### `bl dataset upload` -| Field | Value | -| --------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `dataset upload` | -| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian | -| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | +| Field | Value | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `dataset upload` | +| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian | +| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | #### Flags -| Flag | Type | Required | Description | -| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image/video) | -| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | -| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) | +| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | +| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes -- Supports .jsonl (text) and .zip (audio/image/video archives with a -- data.jsonl manifest). Six record schemas are recognized: chatml = -- {messages:[...]} (SFT); dpo = {messages:[...], chosen, rejected}; -- cpt = {text:"..."} (continual pre-training, raw text); tts = -- {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); image = -- {img_path:"..."} (image generation); video = {first_frame_path:"...", -- video_path:"..."} (video generation). With no --schema, a record -- carrying wav_fn is validated as TTS, img_path as image, video_path / -- first_frame_path as video, chosen/rejected as DPO, text (no messages) -- as CPT, otherwise ChatML. Upload cap: 300MB text, 1GB image/video. -- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so -- the purpose tag is persisted (the DashScope-native /api/v1/files drops it). +- Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl +- manifest). Five record schemas are recognized: chatml = {messages:[...]} +- (SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."} +- (continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav", +- text:"..."} (audio fine-tuning); image = {img_path:"..."} (image +- generation). With no --schema, a record carrying wav_fn is validated as +- TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT, +- otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the +- OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is +- persisted (the DashScope-native /api/v1/files drops it). #### Examples @@ -172,19 +170,19 @@ bl dataset upload --file train.jsonl --no-validate ### `bl dataset validate` -| Field | Value | -| --------------- | ------------------------------------------------------------------------------------------------------ | -| **Name** | `dataset validate` | -| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | -| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | +| Field | Value | +| --------------- | ----------------------------------------------------------------------------------------------- | +| **Name** | `dataset validate` | +| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | +| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | #### Flags -| Flag | Type | Required | Description | -| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | +| Flag | Type | Required | Description | +| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | #### Notes @@ -193,14 +191,13 @@ bl dataset upload --file train.jsonl --no-validate - Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen, - rejected}; cpt = {text:"..."} (continual pre-training, raw text); - tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); -- image = {img_path:"..."} (image generation); video = -- {first_frame_path:"...", video_path:"..."} (video generation). With no -- --schema, a record carrying wav_fn is validated as TTS, img_path as -- image, video_path / first_frame_path as video, chosen/rejected as DPO, -- text (no messages) as CPT, otherwise ChatML. Pass --schema to require a -- specific shape on every record. ZIP archives (.zip) are validated -- structurally (data.jsonl present, media references resolve) in addition -- to per-record content checks. Use --full-validate to JSON.parse every line. +- image = {img_path:"..."} (image generation). With no --schema, a record +- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected +- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to +- require a specific shape on every record. ZIP archives (.zip) are +- validated structurally (data.jsonl present, media references resolve) in +- addition to per-record content checks. Use --full-validate to JSON.parse +- every line. #### Examples diff --git a/skills/bailian-cli/reference/deploy.md b/skills/bailian-cli/reference/deploy.md index 57f4190..184e31f 100644 --- a/skills/bailian-cli/reference/deploy.md +++ b/skills/bailian-cli/reference/deploy.md @@ -29,22 +29,19 @@ Index: [index.md](index.md) #### Flags -| Flag | Type | Required | Description | -| ----------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------ | -| `--model ` | string | yes | Model name (catalog model or fine-tuned output) (required) | -| `--name ` | string | yes | Console display name for the deployment (required) | -| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | -| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | -| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | -| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | -| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | -| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | -| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | -| `--aigc-use-input-prompt ` | boolean | no | Video LoRA (aigc_config): honor the caller's prompt at inference (default false = use preset template) | -| `--aigc-prompt ` | string | no | Video LoRA (aigc_config): preset prompt template used when use-input-prompt is false | -| `--aigc-lora-prompt-default ` | string | no | Video LoRA (aigc_config): default trigger-word phrase for the LoRA | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | +| `--model ` | string | yes | Model name (catalog model or fine-tuned output) (required) | +| `--name ` | string | yes | Console display name for the deployment (required) | +| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | +| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | +| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | +| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | +| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | +| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | +| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes @@ -58,9 +55,6 @@ Index: [index.md](index.md) - Use `bl deploy models --source base` to inspect available templates. - After creation, status starts at PENDING and transitions to RUNNING. - Invoke the deployed model with: bl text chat --model -- For fine-tuned Wan video (i2v/kf2v) LoRA models, use --plan lora and pass -- --aigc-prompt / --aigc-lora-prompt-default (and optionally -- --aigc-use-input-prompt) to set the deployment's aigc_config. - WARNING: --model is overloaded across commands and refers to DIFFERENT - values. `bl deploy create --model` takes the exported model_name (e.g. - `qwen3-8b-ft-...`), but the create response also returns a `deployed_model` @@ -87,10 +81,6 @@ bl deploy create --model qwen3-8b --name my-qwen3-mu --plan mu bl deploy create --model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2 ``` -```bash -bl deploy create --model wan2.5-i2v-preview-ft-xxx --name my-video-lora --plan lora --aigc-prompt "..." --aigc-lora-prompt-default "..." -``` - ### `bl deploy delete` | Field | Value |