From 17f4454df4d46e817c542110855d25906579feef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=95=85=E7=92=83?= Date: Thu, 25 Jun 2026 13:56:15 +0800 Subject: [PATCH] feat: refact validator to support dpo dataset --- packages/cli/src/commands/dataset/upload.ts | 25 +- packages/cli/src/commands/dataset/validate.ts | 28 ++- packages/cli/src/commands/finetune/create.ts | 194 ++++++++++++---- .../cli/tests/e2e/.dataset-dpo-invalid.jsonl | 1 + .../cli/tests/e2e/.dataset-dpo-valid.jsonl | 2 + packages/cli/tests/e2e/dataset.e2e.test.ts | 92 ++++++++ packages/cli/tests/e2e/finetune.e2e.test.ts | 45 ++++ packages/core/src/dataset/index.ts | 2 + packages/core/src/dataset/validate/common.ts | 19 +- packages/core/src/dataset/validate/index.ts | 3 +- packages/core/src/dataset/validate/jsonl.ts | 215 ++++++++++++++---- packages/core/src/dataset/validate/types.ts | 16 ++ packages/core/src/finetune/index.ts | 1 + packages/core/src/finetune/preflight.ts | 79 +++++++ packages/core/tests/dataset-validate.test.ts | 128 +++++++++++ .../core/tests/finetune-preflight.test.ts | 47 ++++ skills/bailian-cli/reference/dataset.md | 67 ++++-- skills/bailian-cli/reference/finetune.md | 3 + 18 files changed, 838 insertions(+), 129 deletions(-) create mode 100644 packages/cli/tests/e2e/.dataset-dpo-invalid.jsonl create mode 100644 packages/cli/tests/e2e/.dataset-dpo-valid.jsonl create mode 100644 packages/core/src/finetune/preflight.ts create mode 100644 packages/core/tests/dataset-validate.test.ts create mode 100644 packages/core/tests/finetune-preflight.test.ts diff --git a/packages/cli/src/commands/dataset/upload.ts b/packages/cli/src/commands/dataset/upload.ts index 25f7f83..a5aefa8 100644 --- a/packages/cli/src/commands/dataset/upload.ts +++ b/packages/cli/src/commands/dataset/upload.ts @@ -3,6 +3,7 @@ import { detectOutputFormat, uploadDataset, validateDataset, + parseDatasetSchemaFlag, MAX_DATASET_BYTES, BailianError, ExitCode, @@ -28,7 +29,8 @@ function formatIssue(issue: ValidationResult["errors"][number]): string { export default defineCommand({ name: "dataset upload", description: "Upload a dataset file (.jsonl) to Bailian", - usage: "bl dataset upload --file [--purpose ] [--no-validate] [--full-validate]", + usage: + "bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]", options: [ { flag: "--file ", @@ -39,6 +41,11 @@ export default defineCommand({ flag: "--purpose ", description: 'Dataset purpose tag (default: "fine-tune"; e.g. "evaluation")', }, + { + flag: "--schema ", + description: + 'Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record.', + }, { flag: "--no-validate", description: "Skip the local JSONL pre-flight check (not recommended)", @@ -52,15 +59,19 @@ export default defineCommand({ ], examples: [ "bl dataset upload --file train.jsonl", + "bl dataset upload --file dpo.jsonl --schema dpo", "bl dataset upload --file eval.jsonl --purpose evaluation", "bl dataset upload --file train.jsonl --full-validate", "bl dataset upload --file train.jsonl --no-validate", ], notes: [ - "Only .jsonl is supported in this release. The default validator expects a", - 'ChatML schema (each line a JSON object with a "messages" array). Other', - "purposes may carry a different schema in the future and would be served", - "by a purpose-specific validator at that point.", + "Only .jsonl is supported in this release. Two record schemas are", + "recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],", + "chosen, rejected} where chosen/rejected are single assistant messages.", + "With no --schema, a record carrying chosen/rejected is validated as DPO;", + "pass --schema dpo to require it on every record, or --schema chatml to", + "ignore preference fields. Other purposes may carry a different schema in", + "the future and would be served by a purpose-specific validator.", "The dataset upload cap is 300MB per file.", "Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so", "the purpose tag is persisted (the DashScope-native /api/v1/files drops it).", @@ -72,10 +83,11 @@ export default defineCommand({ const purpose = (flags.purpose as string | undefined) || "fine-tune"; const skipValidate = Boolean(flags.noValidate); const fullValidate = Boolean(flags.fullValidate); + const schema = parseDatasetSchemaFlag(flags.schema as string | undefined); const format = detectOutputFormat(config.output); if (!skipValidate) { - const result = await validateDataset(filePath!, { fullValidate }); + const result = await validateDataset(filePath!, { fullValidate, schema }); if (!result.valid) { const lines = [ `Dataset validation failed for ${filePath}`, @@ -112,6 +124,7 @@ export default defineCommand({ purpose, max_bytes: MAX_DATASET_BYTES, validate: !skipValidate, + schema: schema ?? "auto", }, format, ); diff --git a/packages/cli/src/commands/dataset/validate.ts b/packages/cli/src/commands/dataset/validate.ts index 0e6a389..3cac884 100644 --- a/packages/cli/src/commands/dataset/validate.ts +++ b/packages/cli/src/commands/dataset/validate.ts @@ -2,6 +2,7 @@ import { defineCommand, detectOutputFormat, validateDataset, + parseDatasetSchemaFlag, BailianError, ExitCode, type Config, @@ -32,7 +33,7 @@ function formatStats(r: ValidationResult): string[] { export default defineCommand({ name: "dataset validate", description: "Locally validate a dataset file (.jsonl) without uploading", - usage: "bl dataset validate --file [--full-validate]", + usage: "bl dataset validate --file [--full-validate] [--schema ]", options: [ { flag: "--file ", description: "Local .jsonl dataset file", required: true }, { @@ -40,16 +41,26 @@ export default defineCommand({ description: "JSON.parse every line instead of sampling (slower)", type: "boolean", }, + { + flag: "--schema ", + description: + 'Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record.', + }, ], examples: [ "bl dataset validate --file train.jsonl", + "bl dataset validate --file dpo.jsonl --schema dpo", "bl dataset validate --file eval.jsonl --full-validate", "bl dataset validate --file train.jsonl --output json", ], notes: [ "Default scan: every line gets a structural check, then ~160 lines (front 50,", "evenly spaced 100, last 10) are JSON.parsed against the active schema.", - "Today the only registered .jsonl schema is ChatML (messages array).", + "Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,", + "rejected} where chosen/rejected are single assistant messages. With no", + "--schema, a record carrying chosen/rejected is validated as DPO; pass", + "--schema dpo to require chosen/rejected on every record (strict), or", + "--schema chatml to ignore preference fields.", "Use --full-validate to JSON.parse every line.", ], async run(config: Config, flags: GlobalFlags) { @@ -57,14 +68,23 @@ export default defineCommand({ if (!filePath) failIfMissing("file", "bl dataset validate --file "); const fullValidate = Boolean(flags.fullValidate); + const schema = parseDatasetSchemaFlag(flags.schema as string | undefined); const format = detectOutputFormat(config.output); if (config.dryRun) { - emitResult({ action: "dataset.validate", file: filePath, full: fullValidate }, format); + emitResult( + { + action: "dataset.validate", + file: filePath, + full: fullValidate, + schema: schema ?? "auto", + }, + format, + ); return; } - const result = await validateDataset(filePath!, { fullValidate }); + const result = await validateDataset(filePath!, { fullValidate, schema }); if (format === "json") { // For json output we always emit the structured result, exit code conveys validity. diff --git a/packages/cli/src/commands/finetune/create.ts b/packages/cli/src/commands/finetune/create.ts index 7632c51..0ea8c4b 100644 --- a/packages/cli/src/commands/finetune/create.ts +++ b/packages/cli/src/commands/finetune/create.ts @@ -7,6 +7,7 @@ import { validateDataset, fetchModelCapability, listSupportedTrainingTypes, + preflightBatchSizeGate, isTrainingTypeCli, toServerTrainingType, TRAINING_TYPES_CLI, @@ -18,6 +19,7 @@ import { type CreateFineTuneRequest, type FineTuneHyperParameters, type DatasetFile, + type DatasetSchema, type ValidationResult, } from "bailian-cli-core"; import { existsSync, statSync } from "fs"; @@ -50,30 +52,44 @@ function formatIssue(issue: ValidationResult["errors"][number]): string { } interface ResolvedDataset { - /** file-ids in input order (local uploads resolved to their new ids). */ + /** + * Tokens in input order. Local paths are kept as-is here (a placeholder + * until `uploadResolvedLocal` swaps them for real file-ids); bare file-ids + * pass through untouched. In dry-run the paths stay (the previewed body + * reflects exactly what the user typed). + */ fileIds: string[]; - /** local paths that were uploaded (empty in dry-run). */ - uploaded: DatasetFile[]; - /** local paths recorded but not uploaded (dry-run only). */ - pendingPaths: string[]; - /** in-hand size for the first token, if known (avoids a redundant getDataset). */ + /** Local paths in input order, for the deferred upload step. */ + localPaths: string[]; + /** In-hand size for the first local token, if known (local statSync). */ firstSize?: number; + /** + * Total training-sample count across local tokens, when known. Sourced from + * `validateDataset`'s `stats.totalRecords` (summed per token). Undefined when + * any token is a bare file-id (no local file to count) or in dry-run — the + * pre-submit batch-size gate only fires when this is known, so file-id flows + * fall through to the platform rather than risk a false positive. + */ + recordCount?: number; } /** - * Resolve a comma-separated `--datasets` / `--validations` value into - * file-ids, uploading any local paths through the same pipeline as - * `bl dataset upload` (validate → upload). File-id tokens are passed through. + * Analyze a comma-separated `--datasets` / `--validations` value WITHOUT + * uploading: bare file-ids pass through; local paths are validated through the + * same pipeline as `bl dataset upload` (so structural errors surface here), + * their sample count and size are captured for the pre-submit gate, and the + * path itself is recorded in `localPaths` for a later, deferred upload. * - * In dry-run mode no upload happens: local paths are recorded in - * `pendingPaths` and left in `fileIds` as-is so the previewed body still - * reflects what the user typed. + * Splitting analysis from upload lets the batch-size gate fire before any + * network call — a doomed job (too few samples) is rejected without burning an + * upload, and is offline-testable. In dry-run mode local paths are not + * validated (the preview never touches the network or the disk beyond stat). */ -async function resolveDatasetTokens( +async function analyzeDatasetTokens( config: Config, raw: string, - purpose: string, label: string, + schema?: DatasetSchema, ): Promise { const tokens = raw .split(",") @@ -84,23 +100,32 @@ async function resolveDatasetTokens( } const fileIds: string[] = []; - const uploaded: DatasetFile[] = []; - const pendingPaths: string[] = []; + const localPaths: string[] = []; let firstSize: number | undefined; + let recordCount: number | undefined; + // A file-id token has no local file to count, so the total sample count is + // only knowable when every token is a local path. Once any file-id is seen, + // flip to unknown and stop accumulating to avoid an undercount that could + // trip the batch-size gate falsely. + let recordCountKnown = true; - for (const [index, token] of tokens.entries()) { + for (const token of tokens) { if (!isLocalPath(token)) { fileIds.push(token); - continue; - } - if (config.dryRun) { - pendingPaths.push(token); - fileIds.push(token); + recordCountKnown = false; continue; } - // Local path → validate then upload (same flow as `bl dataset upload`). - const result = await validateDataset(token); + fileIds.push(token); + localPaths.push(token); + + if (config.dryRun) continue; + + // Local path → validate (same checks as `bl dataset upload`). Upload is + // deferred to `uploadResolvedLocal` so the gate can run first. The schema + // (SFT vs DPO) is derived from --training-type so a DPO job validates the + // chosen/rejected preference pairs here, not on the platform. + const result = await validateDataset(token, { schema }); if (!result.valid) { const lines = [ `Dataset validation failed for ${token}`, @@ -129,6 +154,40 @@ async function resolveDatasetTokens( } } + // Accumulate the sample count so the caller can pre-flight the batch-size + // gate before submitting. `totalRecords` is set by the jsonl validator as + // (non-blank lines); undefined stats fall back to "unknown" (no gate). + const tokenRecords = result.stats.totalRecords; + if (typeof tokenRecords === "number") { + recordCount = (recordCount ?? 0) + tokenRecords; + } + if (firstSize === undefined) firstSize = statSync(token).size; + } + + return { + fileIds, + localPaths, + firstSize, + recordCount: recordCountKnown ? recordCount : undefined, + }; +} + +/** + * Upload each local path recorded in `resolved.localPaths`, swapping the + * placeholder path entries in `resolved.fileIds` for the returned file-ids. + * Returns the uploaded file records (for the confirmation panel). No-op in + * dry-run. Validation already happened in `analyzeDatasetTokens`, so this is + * pure upload. + */ +async function uploadResolvedLocal( + config: Config, + resolved: ResolvedDataset, + purpose: string, + label: string, +): Promise { + const uploaded: DatasetFile[] = []; + for (const [index, token] of resolved.fileIds.entries()) { + if (!isLocalPath(token)) continue; const file: DatasetFile = await uploadDataset(config, { filePath: token, purpose }); if (!file.file_id) { throw new BailianError( @@ -137,17 +196,14 @@ async function resolveDatasetTokens( ); } uploaded.push(file); - fileIds.push(file.file_id); - if (index === 0) firstSize = file.size; - + resolved.fileIds[index] = file.file_id; if (!config.quiet) { process.stderr.write( `Uploaded ${basename(token)} → ${file.file_id} (auto from --${label})\n`, ); } } - - return { fileIds, uploaded, pendingPaths, firstSize }; + return uploaded; } export default defineCommand({ @@ -232,6 +288,9 @@ export default defineCommand({ "--datasets / --validations accept either file-ids (from `bl dataset", "upload`) or local .jsonl paths. Local paths are validated and uploaded", "first, then their file-ids are submitted — a one-step upload-and-train.", + "Pre-submit gate: if the training dataset's sample count is not greater", + "than batch_size, the job is rejected before upload or quota consumption", + "(the platform would otherwise fail ~10 min in, after data processing).", ], async run(config: Config, flags: GlobalFlags) { const model = flags.model as string | undefined; @@ -240,18 +299,11 @@ export default defineCommand({ const datasetsRaw = flags.datasets as string | undefined; if (!datasetsRaw) failIfMissing("datasets", "bl finetune create --datasets "); - const training = await resolveDatasetTokens(config, datasetsRaw!, "fine-tune", "datasets"); - const trainingFileIds = training.fileIds; - - const validationsRaw = flags.validations as string | undefined; - const validation = validationsRaw - ? await resolveDatasetTokens(config, validationsRaw, "fine-tune", "validations") - : undefined; - const validationFileIds = validation?.fileIds; - + // Resolve the training type before analyzing datasets so the validator can + // enforce the right record schema (DPO jobs require chosen/rejected on + // every record). Whitelist is the single source of truth in core + // (TRAINING_TYPES_CLI); any other value is rejected up-front. const trainingType = (flags.trainingType as string | undefined) || DEFAULT_TRAINING_TYPE; - // Whitelist is the single source of truth in core (TRAINING_TYPES_CLI); - // any other value is rejected up-front with an actionable error. if (!isTrainingTypeCli(trainingType)) { throw new BailianError( `--training-type "${trainingType}" is not supported.`, @@ -259,6 +311,18 @@ export default defineCommand({ `Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`, ); } + // dpo / dpo-lora → "dpo" schema (strict chosen/rejected); else ChatML. + const datasetSchema: DatasetSchema = trainingType.startsWith("dpo") ? "dpo" : "chatml"; + + const training = await analyzeDatasetTokens(config, datasetsRaw!, "datasets", datasetSchema); + const trainingFileIds = training.fileIds; + + const validationsRaw = flags.validations as string | undefined; + const validation = validationsRaw + ? await analyzeDatasetTokens(config, validationsRaw, "validations", datasetSchema) + : undefined; + const validationFileIds = validation?.fileIds; + const modelName = flags.modelName as string | undefined; const suffix = flags.suffix as string | undefined; @@ -299,6 +363,48 @@ export default defineCommand({ } } + // Pre-submit batch-size gate: the platform rejects a job whose number of + // training samples is not greater than batch_size, but only surfaces that + // ~10 minutes into the run (after data processing). Fail fast here, before + // burning quota. `recordCount` is only known when every --datasets token + // was a local file we validated; file-id tokens fall through to the + // platform rather than risk a false positive from an undercount. + // + // The decision lives in core (`preflightBatchSizeGate`) — a structured, + // job-level pre-flight that returns a `ValidationIssue` (same shape / stable + // code as `validateDataset`) so the failure surfaces through the same + // `BailianError` + issue convention used by `bl dataset upload`/`validate`. + // ExitCode.GENERAL matches the existing validation-failed exit code. + if (!config.dryRun && training.recordCount !== undefined) { + // 16 is the platform default when neither the user nor the small-file + // auto-adjust set a batch_size (see the auto-adjust comment above). + const effectiveBatchSize = hp.batch_size ?? 16; + const gate = preflightBatchSizeGate({ + recordCount: training.recordCount, + batchSize: effectiveBatchSize, + }); + if (!gate.ok && gate.issue) { + throw new BailianError(gate.issue.message, ExitCode.GENERAL, gate.hint); + } + } + + // Upload local paths now that the gate has cleared them. This swaps the + // placeholder path entries in `training.fileIds` / `validation?.fileIds` + // for real file-ids, so the body and confirmation panel below see ids. + let uploadedTraining: DatasetFile[] = []; + let uploadedValidation: DatasetFile[] = []; + if (!config.dryRun) { + uploadedTraining = await uploadResolvedLocal(config, training, "fine-tune", "datasets"); + if (validation) { + uploadedValidation = await uploadResolvedLocal( + config, + validation, + "fine-tune", + "validations", + ); + } + } + const body: CreateFineTuneRequest = { model: model!, training_file_ids: trainingFileIds, @@ -316,8 +422,8 @@ export default defineCommand({ if (config.dryRun) { const pending = [ - ...training.pendingPaths.map((path) => ({ field: "datasets", path })), - ...(validation?.pendingPaths ?? []).map((path) => ({ field: "validations", path })), + ...training.localPaths.map((path) => ({ field: "datasets", path })), + ...(validation?.localPaths ?? []).map((path) => ({ field: "validations", path })), ]; emitResult( pending.length > 0 @@ -353,10 +459,10 @@ export default defineCommand({ if (validationFileIds) { process.stderr.write(` Validation: ${validationFileIds.join(", ")}\n`); } - for (const file of training.uploaded) { + for (const file of uploadedTraining) { process.stderr.write(` Uploaded: ${file.name} → ${file.file_id}\n`); } - for (const file of validation?.uploaded ?? []) { + for (const file of uploadedValidation) { process.stderr.write(` Uploaded: ${file.name} → ${file.file_id} (validation)\n`); } process.stderr.write(` n_epochs: ${hp.n_epochs}\n`); diff --git a/packages/cli/tests/e2e/.dataset-dpo-invalid.jsonl b/packages/cli/tests/e2e/.dataset-dpo-invalid.jsonl new file mode 100644 index 0000000..cd080c1 --- /dev/null +++ b/packages/cli/tests/e2e/.dataset-dpo-invalid.jsonl @@ -0,0 +1 @@ +{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}} diff --git a/packages/cli/tests/e2e/.dataset-dpo-valid.jsonl b/packages/cli/tests/e2e/.dataset-dpo-valid.jsonl new file mode 100644 index 0000000..4d336e3 --- /dev/null +++ b/packages/cli/tests/e2e/.dataset-dpo-valid.jsonl @@ -0,0 +1,2 @@ +{"messages":[{"role":"user","content":"你能帮我写一篇文章吗?"}],"chosen":{"role":"assistant","content":"当然可以，请告诉我具体方向。"},"rejected":{"role":"assistant","content":"可以。"}} +{"messages":[{"role":"user","content":"安排一下明天的日程?"}],"chosen":{"role":"assistant","content":"当然，请告诉我具体事项。"},"rejected":{"role":"assistant","content":"好的。"}} diff --git a/packages/cli/tests/e2e/dataset.e2e.test.ts b/packages/cli/tests/e2e/dataset.e2e.test.ts index 26473de..38c497e 100644 --- a/packages/cli/tests/e2e/dataset.e2e.test.ts +++ b/packages/cli/tests/e2e/dataset.e2e.test.ts @@ -79,6 +79,98 @@ describe("e2e: dataset (offline)", () => { expect(data.action).toBe("dataset.upload"); expect(data.validate).toBe(false); }); + + test("dataset validate 自动识别 DPO 并校验 chosen/rejected", async () => { + // No --schema: a record carrying chosen/rejected is auto-detected as DPO + // and the valid fixture passes. + const file = join(__dirname, ".dataset-dpo-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "validate", + "--file", + file, + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ valid: boolean; stats: { totalRecords?: number } }>(stdout); + expect(data.valid).toBe(true); + expect(data.stats.totalRecords).toBe(2); + }); + + test("dataset validate --schema dpo 拒绝缺失 rejected 的记录", async () => { + const file = join(__dirname, ".dataset-dpo-invalid.jsonl"); + const { stdout, exitCode } = await runCli([ + "dataset", + "validate", + "--file", + file, + "--schema", + "dpo", + "--output", + "json", + ]); + expect(exitCode).not.toBe(0); + const data = parseStdoutJson<{ valid: boolean; errors: { code: string; path?: string }[] }>( + stdout, + ); + expect(data.valid).toBe(false); + expect(data.errors.map((e) => e.code)).toContain("MISSING_REJECTED"); + }); + + test("dataset validate --schema chatml 忽略 chosen/rejected（不报 DPO 错误）", async () => { + // Same invalid-DPO file, but --schema chatml must not run DPO checks. + const file = join(__dirname, ".dataset-dpo-invalid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "validate", + "--file", + file, + "--schema", + "chatml", + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ valid: boolean; errors: { code: string }[] }>(stdout); + expect(data.valid).toBe(true); + expect(data.errors.filter((c) => c.code.startsWith("MISSING_"))).toEqual([]); + }); + + test("dataset validate --schema 以非零码退出", async () => { + const file = join(__dirname, ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "validate", + "--file", + file, + "--schema", + "sft", + "--output", + "json", + ]); + expect(exitCode).not.toBe(0); + expect(`${stdout}\n${stderr}`).toMatch(/Unsupported --schema/); + }); + + test("dataset upload --dry-run 转发 --schema", async () => { + const file = join(__dirname, ".dataset-dpo-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "dataset", + "upload", + "--file", + file, + "--schema", + "dpo", + "--dry-run", + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ action: string; schema: string }>(stdout); + expect(data.action).toBe("dataset.upload"); + expect(data.schema).toBe("dpo"); + }); }); describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => { diff --git a/packages/cli/tests/e2e/finetune.e2e.test.ts b/packages/cli/tests/e2e/finetune.e2e.test.ts index a66a503..1dd21ab 100644 --- a/packages/cli/tests/e2e/finetune.e2e.test.ts +++ b/packages/cli/tests/e2e/finetune.e2e.test.ts @@ -177,6 +177,51 @@ describe("e2e: finetune (offline)", () => { expect(exitCode, stdout + stderr).not.toBe(0); }); + test("finetune create 样本数 <= batch_size 时提交前快速失败且不上传", async () => { + // The fixture has 3 records; the small-file auto-adjust sets batch_size=8, + // so 3 <= 8 trips the pre-submit gate. The gate fires before any upload, + // so this is fully offline (no key, no network) — the proof is that the + // error is the gate message AND no "Uploaded …" line ever appears. + const localPath = join(cliPackageRoot, "tests", "e2e", ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "finetune", + "create", + "--model", + "qwen3-8b", + "--datasets", + localPath, + "--yes", + "--output", + "json", + ]); + expect(exitCode, stdout + stderr).not.toBe(0); + const combined = `${stdout}\n${stderr}`; + expect(combined).toMatch(/not greater than batch_size/i); + // Crucially, no upload happened — the gate must fire before the upload step. + expect(combined).not.toMatch(/Uploaded .* → file-/); + }); + + test("finetune create --batch-size 过小仍按 8 下限比较（不绕过卡口）", async () => { + // Even with --batch-size 1 (server clamps to 8), 3 samples <= 8 still trips + // the gate — confirms the gate uses the clamped/effective batch, not the raw. + const localPath = join(cliPackageRoot, "tests", "e2e", ".dataset-valid.jsonl"); + const { stdout, stderr, exitCode } = await runCli([ + "finetune", + "create", + "--model", + "qwen3-8b", + "--datasets", + localPath, + "--batch-size", + "1", + "--yes", + "--output", + "json", + ]); + expect(exitCode, stdout + stderr).not.toBe(0); + expect(`${stdout}\n${stderr}`).toMatch(/batch_size \(8\)/); + }); + test.each([ ["list", ["--status", "RUNNING"]], ["get", ["--job-id", "ft-xxx"]], diff --git a/packages/core/src/dataset/index.ts b/packages/core/src/dataset/index.ts index 6d930ba..91f31b8 100644 --- a/packages/core/src/dataset/index.ts +++ b/packages/core/src/dataset/index.ts @@ -6,10 +6,12 @@ export { registerValidator, listSupportedFormats, MAX_DATASET_BYTES, + parseDatasetSchemaFlag, } from "./validate/index.ts"; export type { ValidatorSpec, ValidateOpts, + DatasetSchema, ValidationResult, ValidationIssue, ValidationSeverity, diff --git a/packages/core/src/dataset/validate/common.ts b/packages/core/src/dataset/validate/common.ts index 1057b84..6a79537 100644 --- a/packages/core/src/dataset/validate/common.ts +++ b/packages/core/src/dataset/validate/common.ts @@ -9,7 +9,7 @@ import { existsSync, statSync } from "fs"; import { extname } from "path"; import { BailianError } from "../../errors/base.ts"; import { ExitCode } from "../../errors/codes.ts"; -import type { ValidationIssue, ValidationStats } from "./types.ts"; +import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts"; /** * The platform caps dataset uploads at 300MB per file. `bl dataset upload` @@ -66,6 +66,23 @@ export function emptyStats(): ValidationStats { return {}; } +/** + * Parse a `--schema` CLI value into a `DatasetSchema` (or `undefined` for + * auto-detect). Single source of truth for the schema vocabulary so `dataset + * validate`, `dataset upload`, and any future caller agree on accepted values + * and error wording. Throws USAGE for anything unrecognized. + */ +export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined { + if (value === undefined || value.trim() === "") return undefined; + const v = value.trim(); + if (v === "chatml" || v === "dpo") return v; + throw new BailianError( + `Unsupported --schema "${value}". Supported: chatml, dpo.`, + ExitCode.USAGE, + `Omit --schema to auto-detect per record (a record with chosen/rejected is treated as DPO).`, + ); +} + /** Produce a deterministic set of sample line indices for deep checking. * Indices are 1-based to match what users see in editors / error messages. * diff --git a/packages/core/src/dataset/validate/index.ts b/packages/core/src/dataset/validate/index.ts index 657df8d..67bf34f 100644 --- a/packages/core/src/dataset/validate/index.ts +++ b/packages/core/src/dataset/validate/index.ts @@ -4,10 +4,11 @@ export { registerValidator, listSupportedFormats, } from "./registry.ts"; -export { MAX_DATASET_BYTES } from "./common.ts"; +export { MAX_DATASET_BYTES, parseDatasetSchemaFlag } from "./common.ts"; export type { ValidatorSpec, ValidateOpts, + DatasetSchema, ValidationResult, ValidationIssue, ValidationSeverity, diff --git a/packages/core/src/dataset/validate/jsonl.ts b/packages/core/src/dataset/validate/jsonl.ts index 8fb0cd9..7889431 100644 --- a/packages/core/src/dataset/validate/jsonl.ts +++ b/packages/core/src/dataset/validate/jsonl.ts @@ -1,13 +1,16 @@ /** * JSONL validator for ChatML-style datasets (e.g. SFT training data). * - * Schema scope: each line is `{"messages": [{role, content}, ...]}` with - * roles in (system, user, assistant). This matches the platform's documented - * SFT training format. Other JSONL schemas (e.g. evaluation datasets with - * different field shapes) should ship their own validator and register it — - * the registry can be extended in the future to dispatch on `(extension, - * purpose)` rather than extension alone if a purpose-specific .jsonl schema - * appears. + * Schema scope: the `.jsonl` ChatML family. Two record shapes are recognized: + * - SFT: `{"messages": [{role, content}, ...]}` + * - DPO: `{"messages": [...], "chosen": {role, content}, "rejected": {...}}` + * `chosen`/`rejected` are single assistant messages — the preferred vs + * dispreferred response. Which shape is enforced is selected by + * `ValidateOpts.schema` (`"chatml"` | `"dpo"`), defaulting to per-record + * auto-detect. Other JSONL schemas (e.g. evaluation datasets with a different + * field shape) should ship their own validator and register it — the registry + * can be extended in the future to dispatch on `(extension, purpose)` rather + * than extension alone if a purpose-specific .jsonl schema appears. * * Two-stage strategy (see decision log): * 1. Quick scan — readline pass over the entire file checking only that @@ -21,7 +24,13 @@ */ import { createReadStream } from "fs"; import { createInterface } from "readline"; -import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts"; +import type { + ValidatorSpec, + ValidateOpts, + ValidationResult, + ValidationIssue, + DatasetSchema, +} from "./types.ts"; import { makeIssue, pickSampleLines } from "./common.ts"; const VALID_ROLES = new Set(["system", "user", "assistant"]); @@ -76,6 +85,7 @@ async function deepCheck( filePath: string, totalLines: number, fullValidate: boolean, + schema: DatasetSchema | undefined, signal?: AbortSignal, ): Promise { const targetSet = fullValidate ? null : new Set(pickSampleLines(totalLines)); @@ -108,30 +118,98 @@ async function deepCheck( continue; } - issues.push(...inspectChatMLRecord(obj, lineNo)); + issues.push(...inspectRecord(obj, lineNo, schema)); } return { sampled, issues }; } /** - * Validate one ChatML record. Hard errors are returned with severity "error", - * advisory checks (role ordering) as "warning". Caller dedupes/aggregates. + * Structural checks for a single message object `{role, content}`. Shared by + * the `messages[]` entries and the DPO `chosen` / `rejected` preference fields + * (which are each a single assistant message). Caller-supplied `path` scopes + * the issue location (e.g. `messages[2]` vs `chosen`). */ -function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] { +function inspectMessageObject(msg: unknown, lineNo: number, path: string): ValidationIssue[] { const out: ValidationIssue[] = []; - if (obj === null || typeof obj !== "object" || Array.isArray(obj)) { + if (msg === null || typeof msg !== "object" || Array.isArray(msg)) { out.push( + makeIssue("error", "MESSAGE_NOT_OBJECT", `Message must be an object.`, { + line: lineNo, + path, + }), + ); + return out; + } + const m = msg as Record; + const role = m.role; + const content = m.content; + if (typeof role !== "string" || !VALID_ROLES.has(role)) { + out.push( + makeIssue( + "error", + "INVALID_ROLE", + `Invalid role "${String(role)}". Expected one of: system, user, assistant.`, + { line: lineNo, path: `${path}.role` }, + ), + ); + } + if (typeof content !== "string") { + out.push( + makeIssue("error", "INVALID_CONTENT", `"content" must be a string (got ${typeof content}).`, { + line: lineNo, + path: `${path}.content`, + }), + ); + } + return out; +} + +/** + * Dispatch one record to the right schema inspector. + * + * SFT and DPO are not sibling schemas — DPO is a *superset* of SFT + * (`{messages:[...], chosen, rejected}` = the ChatML prompt + a preference + * pair). So this dispatcher only decides *whether* to also validate the + * preference fields; the `messages[]` core is always handled by + * `inspectChatMLRecord` (DPO calls into it). + * + * Schema selection mirrors the `ValidateOpts.schema` contract: + * - `"chatml"` → SFT only (preference fields ignored). + * - `"dpo"` → DPO, strictly (every record must carry chosen+rejected). + * - `undefined` (auto) → per record: DPO when `chosen`/`rejected` present, else SFT. + */ +function inspectRecord(obj: unknown, lineNo: number, schema?: DatasetSchema): ValidationIssue[] { + if (obj === null || typeof obj !== "object" || Array.isArray(obj)) { + return [ makeIssue( "error", "RECORD_NOT_OBJECT", `Each line must be a JSON object, got ${Array.isArray(obj) ? "array" : typeof obj}.`, { line: lineNo }, ), - ); - return out; + ]; } - const record = obj as Record; + const hasChosen = "chosen" in record; + const hasRejected = "rejected" in record; + const isDpo = schema === "dpo" || (schema === undefined && (hasChosen || hasRejected)); + return isDpo + ? inspectDPORecord(record, lineNo, hasChosen, hasRejected) + : inspectChatMLRecord(record, lineNo); +} + +/** + * SFT (ChatML) record: `{"messages": [{role, content}, ...]}`. + * + * Validates the shared `messages[]` core that every ChatML-family record + * carries — including DPO, which is why `inspectDPORecord` delegates here for + * the prompt portion. `chosen`/`rejected`, if present on the record, are + * intentionally ignored: callers wanting those checked must go through DPO + * mode. Hard errors return as "error", advisory role-ordering checks as + * "warning". + */ +function inspectChatMLRecord(record: Record, lineNo: number): ValidationIssue[] { + const out: ValidationIssue[] = []; const messages = record.messages; if (!Array.isArray(messages)) { out.push( @@ -159,38 +237,8 @@ function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] { for (let i = 0; i < messages.length; i++) { const msg = messages[i]; const path = `messages[${i}]`; - if (msg === null || typeof msg !== "object" || Array.isArray(msg)) { - out.push( - makeIssue("error", "MESSAGE_NOT_OBJECT", `Message must be an object.`, { - line: lineNo, - path, - }), - ); - continue; - } - const m = msg as Record; - const role = m.role; - const content = m.content; - if (typeof role !== "string" || !VALID_ROLES.has(role)) { - out.push( - makeIssue( - "error", - "INVALID_ROLE", - `Invalid role "${String(role)}". Expected one of: system, user, assistant.`, - { line: lineNo, path: `${path}.role` }, - ), - ); - } - if (typeof content !== "string") { - out.push( - makeIssue( - "error", - "INVALID_CONTENT", - `"content" must be a string (got ${typeof content}).`, - { line: lineNo, path: `${path}.content` }, - ), - ); - } + out.push(...inspectMessageObject(msg, lineNo, path)); + const role = (msg as Record | null)?.role; if (role === "system") { if (i !== 0) { @@ -238,6 +286,76 @@ function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] { return out; } +/** + * DPO record: `{"messages": [...], "chosen": {role, content}, "rejected": {...}}`. + * + * The prompt context (`messages[]`) is validated by `inspectChatMLRecord`; + * this function adds the preference pair on top. `chosen`/`rejected` are each a + * single assistant message — the preferred vs dispreferred response — so they + * reuse `inspectMessageObject` with a scoped `path`. + * + * If the prompt is structurally broken (missing/empty `messages`), the SFT + * inspector already reported the hard error and we skip preference checks — a + * record missing its prompt is too broken to meaningfully check chosen/rejected + * on top, matching the original early-return semantics. + */ +function inspectDPORecord( + record: Record, + lineNo: number, + hasChosen: boolean, + hasRejected: boolean, +): ValidationIssue[] { + const out = inspectChatMLRecord(record, lineNo); + const messages = record.messages; + if (!Array.isArray(messages) || messages.length === 0) return out; + + if (!hasChosen) { + out.push( + makeIssue("error", "MISSING_CHOSEN", `DPO record is missing the "chosen" preference.`, { + line: lineNo, + path: "chosen", + }), + ); + } + if (!hasRejected) { + out.push( + makeIssue("error", "MISSING_REJECTED", `DPO record is missing the "rejected" preference.`, { + line: lineNo, + path: "rejected", + }), + ); + } + if (hasChosen) { + out.push(...inspectMessageObject(record.chosen, lineNo, "chosen")); + const role = (record.chosen as Record | null)?.role; + if (typeof role === "string" && role !== "assistant") { + out.push( + makeIssue( + "warning", + "PREFERENCE_ROLE_NOT_ASSISTANT", + `"chosen" role should be "assistant" (got "${role}").`, + { line: lineNo, path: "chosen.role" }, + ), + ); + } + } + if (hasRejected) { + out.push(...inspectMessageObject(record.rejected, lineNo, "rejected")); + const role = (record.rejected as Record | null)?.role; + if (typeof role === "string" && role !== "assistant") { + out.push( + makeIssue( + "warning", + "PREFERENCE_ROLE_NOT_ASSISTANT", + `"rejected" role should be "assistant" (got "${role}").`, + { line: lineNo, path: "rejected.role" }, + ), + ); + } + } + return out; +} + export const jsonlValidator: ValidatorSpec = { format: "jsonl", extensions: [".jsonl"], @@ -280,6 +398,7 @@ export const jsonlValidator: ValidatorSpec = { filePath, quick.totalLines, Boolean(opts.fullValidate), + opts.schema, opts.signal, ); diff --git a/packages/core/src/dataset/validate/types.ts b/packages/core/src/dataset/validate/types.ts index a44c9f6..a5a3a86 100644 --- a/packages/core/src/dataset/validate/types.ts +++ b/packages/core/src/dataset/validate/types.ts @@ -16,8 +16,24 @@ export interface ValidateOpts { maxBytes?: number; /** Optional abort signal for long-running scans. */ signal?: AbortSignal; + /** + * Record-schema selector for formats that carry more than one schema under + * the same extension. Today only the `.jsonl` ChatML family honors it: + * - `"chatml"` — `{messages: [...]}` (SFT). `chosen`/`rejected` ignored. + * - `"dpo"` — `{messages: [...], chosen: {role,content}, rejected: {...}}`. + * Every record MUST carry `chosen` + `rejected`. + * - `undefined` — auto-detect per record: a record with `chosen` or + * `rejected` is validated as DPO, otherwise as ChatML. + * `finetune create` sets this from `--training-type` (dpo* → "dpo") so a DPO + * job with malformed preference pairs fails at validate time, not on the + * platform ten minutes in. + */ + schema?: DatasetSchema; } +/** The schemas a `.jsonl` record can be validated against. */ +export type DatasetSchema = "chatml" | "dpo"; + export type ValidationSeverity = "error" | "warning"; export interface ValidationIssue { diff --git a/packages/core/src/finetune/index.ts b/packages/core/src/finetune/index.ts index 8936680..b162966 100644 --- a/packages/core/src/finetune/index.ts +++ b/packages/core/src/finetune/index.ts @@ -1,3 +1,4 @@ export * from "./types.ts"; export * from "./api.ts"; export * from "./capability.ts"; +export * from "./preflight.ts"; diff --git a/packages/core/src/finetune/preflight.ts b/packages/core/src/finetune/preflight.ts new file mode 100644 index 0000000..ac48360 --- /dev/null +++ b/packages/core/src/finetune/preflight.ts @@ -0,0 +1,79 @@ +/** + * Finetune job-level pre-flight checks. + * + * Sibling to `capability.ts` (the model-capability pre-flight). These checks + * are NOT dataset-format validations — they consume the per-file validation + * output (e.g. `stats.totalRecords` from `validateDataset`) together with + * job-level inputs (hyper-parameters) and decide whether a job is submittable. + * Format/structure checks live in `dataset/validate/`; these live here because + * they depend on concerns the format validators must never know about. + * + * Consistency with the validate architecture: a failing check returns a + * `ValidationIssue` (same shape, stable `code`, `error` severity) so callers + * surface it through the same `BailianError` + issue-list convention used by + * `bl dataset upload` / `bl dataset validate`. The trigger stays inline in + * `finetune create` (the only call site today) — that's the job-level boundary. + */ +import type { ValidationIssue } from "../dataset/validate/types.ts"; + +/** Stable issue code for "too few training samples for the batch size". */ +export const INSUFFICIENT_SAMPLES_CODE = "INSUFFICIENT_SAMPLES"; + +export interface BatchSizeGateInput { + /** + * Total training-sample count across all `--datasets` files. Sourced from + * `validateDataset`'s `stats.totalRecords` (summed per file). The gate only + * fires when this is known — i.e. every dataset token was a local file that + * was validated; bare file-id tokens yield no count and fall through to the + * platform. + */ + recordCount: number; + /** + * Effective batch_size the job will run with — after the CLI's clamp + * ([8, 1024]) and small-file auto-adjust, or the platform default (16) when + * neither the user nor auto-adjust set one. + */ + batchSize: number; +} + +export interface BatchSizeGateResult { + ok: boolean; + /** Present when `!ok`, in the same shape `validateDataset` issues use. */ + issue?: ValidationIssue; + /** Actionable guidance; callers surface it as the `BailianError` detail. */ + hint?: string; +} + +/** + * Pre-flight the platform's "training samples must exceed batch_size" rule. + * + * The platform rejects a job whose number of training samples is not greater + * than batch_size, but only surfaces that ~10 minutes into the run (after data + * processing). This gate fails fast, before upload or quota consumption. + * + * Conservative by design — never false-positives: with the platform's default + * 0.9 train split, training samples = 0.9 * recordCount <= recordCount, so + * `recordCount <= batchSize` implies training samples <= batchSize implies + * certain platform failure. Borderline counts (records just above batchSize) + * may still fail on the platform; that's an acceptable false negative for a + * pre-check, and the hint nudges users to leave margin for the split. + */ +export function preflightBatchSizeGate(input: BatchSizeGateInput): BatchSizeGateResult { + const { recordCount, batchSize } = input; + if (recordCount > batchSize) return { ok: true }; + return { + ok: false, + issue: { + severity: "error", + code: INSUFFICIENT_SAMPLES_CODE, + message: `Training dataset has ${recordCount} sample(s), which is not greater than batch_size (${batchSize}).`, + }, + hint: [ + "The platform requires the number of training samples to exceed batch_size.", + "Options:", + " • add more data (recommended: comfortably more than batch_size, since the", + " platform also holds back a default 0.9 train split),", + " • lower --batch-size (server clamps to a minimum of 8).", + ].join("\n"), + }; +} diff --git a/packages/core/tests/dataset-validate.test.ts b/packages/core/tests/dataset-validate.test.ts new file mode 100644 index 0000000..207760e --- /dev/null +++ b/packages/core/tests/dataset-validate.test.ts @@ -0,0 +1,128 @@ +import { afterAll, describe, expect, test } from "vite-plus/test"; +import { mkdirSync, rmSync, writeFileSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { validateDataset, parseDatasetSchemaFlag } from "../src/index.ts"; + +const tmp = join(tmpdir(), `bl-dpo-test-${process.pid}`); +mkdirSync(tmp, { recursive: true }); + +function file(name: string, lines: string[]): string { + const p = join(tmp, name); + writeFileSync(p, lines.join("\n")); + return p; +} + +const DPO_OK = + '{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}'; +const SFT_OK = + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}]}'; + +afterAll(() => rmSync(tmp, { recursive: true, force: true })); + +function codes(r: { errors: { code: string }[]; warnings: { code: string }[] }) { + return { + errors: r.errors.map((e) => e.code), + warnings: r.warnings.map((w) => w.code), + }; +} + +describe("validateDataset — DPO schema", () => { + test("valid DPO record passes under auto-detect and --schema dpo", async () => { + const p = file("ok.jsonl", [DPO_OK]); + const auto = await validateDataset(p, { fullValidate: true }); + expect(auto.valid).toBe(true); + const dpo = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(dpo.valid).toBe(true); + }); + + test("missing rejected → MISSING_REJECTED (auto-detect, since chosen present)", async () => { + const p = file("miss_rej.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("MISSING_REJECTED"); + expect(codes(r).errors).not.toContain("MISSING_CHOSEN"); + }); + + test("missing chosen → MISSING_CHOSEN (auto-detect, since rejected present)", async () => { + const p = file("miss_chosen.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("MISSING_CHOSEN"); + }); + + test('schema "dpo" requires both chosen and rejected on every record', async () => { + // A record with neither chosen nor rejected is SFT-shaped; under --schema dpo + // it must be flagged as missing both preferences. + const p = file("sft_under_dpo.jsonl", [SFT_OK]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toEqual(expect.arrayContaining(["MISSING_CHOSEN", "MISSING_REJECTED"])); + }); + + test('schema "chatml" ignores chosen/rejected (no DPO errors)', async () => { + const p = file("miss_rej_chatml.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "chatml" }); + expect(r.valid).toBe(true); + expect(codes(r).errors.filter((c) => c.startsWith("MISSING_"))).toEqual([]); + }); + + test("SFT-only file under auto-detect is unaffected (no DPO checks)", async () => { + const p = file("sft.jsonl", [SFT_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).toEqual([]); + }); + + test("chosen not a message object → MESSAGE_NOT_OBJECT at path chosen", async () => { + const p = file("bad_chosen.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"chosen":"nope","rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + const err = r.errors.find((e) => e.code === "MESSAGE_NOT_OBJECT"); + expect(err).toBeDefined(); + expect(err!.path).toBe("chosen"); + }); + + test("chosen role=user → PREFERENCE_ROLE_NOT_ASSISTANT warning", async () => { + const p = file("role_warn.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"user","content":"good"},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("PREFERENCE_ROLE_NOT_ASSISTANT"); + }); + + test("multi-turn prompt in messages still validates with DPO preferences", async () => { + const p = file("multiturn.jsonl", [ + '{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"},{"role":"user","content":"c"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(true); + }); +}); + +describe("parseDatasetSchemaFlag", () => { + test("undefined / empty → undefined (auto)", () => { + expect(parseDatasetSchemaFlag(undefined)).toBeUndefined(); + expect(parseDatasetSchemaFlag("")).toBeUndefined(); + expect(parseDatasetSchemaFlag(" ")).toBeUndefined(); + }); + + test("chatml / dpo pass through", () => { + expect(parseDatasetSchemaFlag("chatml")).toBe("chatml"); + expect(parseDatasetSchemaFlag("dpo")).toBe("dpo"); + expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo"); + }); + + test("unrecognized throws", () => { + expect(() => parseDatasetSchemaFlag("sft")).toThrow(/Unsupported --schema/); + }); +}); diff --git a/packages/core/tests/finetune-preflight.test.ts b/packages/core/tests/finetune-preflight.test.ts new file mode 100644 index 0000000..1bc2552 --- /dev/null +++ b/packages/core/tests/finetune-preflight.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "vite-plus/test"; +import { preflightBatchSizeGate, INSUFFICIENT_SAMPLES_CODE } from "../src/index.ts"; + +describe("preflightBatchSizeGate", () => { + test("passes when recordCount exceeds batch_size", () => { + const r = preflightBatchSizeGate({ recordCount: 9, batchSize: 8 }); + expect(r.ok).toBe(true); + expect(r.issue).toBeUndefined(); + expect(r.hint).toBeUndefined(); + }); + + test("passes at the boundary just above batch_size (9 > 8)", () => { + expect(preflightBatchSizeGate({ recordCount: 9, batchSize: 8 }).ok).toBe(true); + // A comfortably-large dataset is fine too. + expect(preflightBatchSizeGate({ recordCount: 1000, batchSize: 16 }).ok).toBe(true); + }); + + test("fails when recordCount equals batch_size (must be *greater than*)", () => { + const r = preflightBatchSizeGate({ recordCount: 8, batchSize: 8 }); + expect(r.ok).toBe(false); + expect(r.issue).toBeDefined(); + expect(r.issue!.severity).toBe("error"); + expect(r.issue!.code).toBe(INSUFFICIENT_SAMPLES_CODE); + expect(r.issue!.message).toMatch(/not greater than batch_size \(8\)/); + expect(r.hint).toMatch(/add more data/); + }); + + test("fails when recordCount is below batch_size (the 3-sample / batch-8 case)", () => { + const r = preflightBatchSizeGate({ recordCount: 3, batchSize: 8 }); + expect(r.ok).toBe(false); + expect(r.issue!.message).toMatch(/3 sample\(s\)/); + expect(r.issue!.message).toMatch(/batch_size \(8\)/); + expect(r.hint).toMatch(/lower --batch-size/); + }); + + test("hint references the 0.9 train split so users leave margin", () => { + const r = preflightBatchSizeGate({ recordCount: 5, batchSize: 8 }); + expect(r.hint).toMatch(/0\.9 train split/); + }); + + test("honors the effective (clamped) batch size, not a raw sub-minimum", () => { + // The CLI clamps --batch-size 1 up to 8 before calling; 3 <= 8 still fails. + const r = preflightBatchSizeGate({ recordCount: 3, batchSize: 8 }); + expect(r.ok).toBe(false); + expect(r.issue!.message).toMatch(/batch_size \(8\)/); + }); +}); diff --git a/skills/bailian-cli/reference/dataset.md b/skills/bailian-cli/reference/dataset.md index 9c3ca74..73078c0 100644 --- a/skills/bailian-cli/reference/dataset.md +++ b/skills/bailian-cli/reference/dataset.md @@ -102,27 +102,31 @@ bl dataset list --output json ### `bl dataset upload` -| Field | Value | -| --------------- | -------------------------------------------------------------------------------------- | -| **Name** | `dataset upload` | -| **Description** | Upload a dataset file (.jsonl) to Bailian | -| **Usage** | `bl dataset upload --file [--purpose ] [--no-validate] [--full-validate]` | +| Field | Value | +| --------------- | --------------------------------------------------------------------------------------------------------------- | +| **Name** | `dataset upload` | +| **Description** | Upload a dataset file (.jsonl) to Bailian | +| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | #### Options -| Flag | Type | Required | Description | -| ------------------ | ------- | -------- | ------------------------------------------------------------- | -| `--file ` | string | yes | Local .jsonl dataset file (≤300MB) | -| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | -| `--no-validate` | boolean | no | Skip the local JSONL pre-flight check (not recommended) | -| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) | +| Flag | Type | Required | Description | +| ------------------ | ------- | -------- | --------------------------------------------------------------------------------------------------- | +| `--file ` | string | yes | Local .jsonl dataset file (≤300MB) | +| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | +| `--schema ` | string | no | Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record. | +| `--no-validate` | boolean | no | Skip the local JSONL pre-flight check (not recommended) | +| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) | #### Notes -- Only .jsonl is supported in this release. The default validator expects a -- ChatML schema (each line a JSON object with a "messages" array). Other -- purposes may carry a different schema in the future and would be served -- by a purpose-specific validator at that point. +- Only .jsonl is supported in this release. Two record schemas are +- recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...], +- chosen, rejected} where chosen/rejected are single assistant messages. +- With no --schema, a record carrying chosen/rejected is validated as DPO; +- pass --schema dpo to require it on every record, or --schema chatml to +- ignore preference fields. Other purposes may carry a different schema in +- the future and would be served by a purpose-specific validator. - The dataset upload cap is 300MB per file. - Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so - the purpose tag is persisted (the DashScope-native /api/v1/files drops it). @@ -133,6 +137,10 @@ bl dataset list --output json bl dataset upload --file train.jsonl ``` +```bash +bl dataset upload --file dpo.jsonl --schema dpo +``` + ```bash bl dataset upload --file eval.jsonl --purpose evaluation ``` @@ -147,24 +155,29 @@ bl dataset upload --file train.jsonl --no-validate ### `bl dataset validate` -| Field | Value | -| --------------- | ---------------------------------------------------------- | -| **Name** | `dataset validate` | -| **Description** | Locally validate a dataset file (.jsonl) without uploading | -| **Usage** | `bl dataset validate --file [--full-validate]` | +| Field | Value | +| --------------- | ------------------------------------------------------------------------------ | +| **Name** | `dataset validate` | +| **Description** | Locally validate a dataset file (.jsonl) without uploading | +| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | #### Options -| Flag | Type | Required | Description | -| ----------------- | ------- | -------- | -------------------------------------------------- | -| `--file ` | string | yes | Local .jsonl dataset file | -| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) | +| Flag | Type | Required | Description | +| ----------------- | ------- | -------- | --------------------------------------------------------------------------------------------------- | +| `--file ` | string | yes | Local .jsonl dataset file | +| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) | +| `--schema ` | string | no | Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record. | #### Notes - Default scan: every line gets a structural check, then ~160 lines (front 50, - evenly spaced 100, last 10) are JSON.parsed against the active schema. -- Today the only registered .jsonl schema is ChatML (messages array). +- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen, +- rejected} where chosen/rejected are single assistant messages. With no +- --schema, a record carrying chosen/rejected is validated as DPO; pass +- --schema dpo to require chosen/rejected on every record (strict), or +- --schema chatml to ignore preference fields. - Use --full-validate to JSON.parse every line. #### Examples @@ -173,6 +186,10 @@ bl dataset upload --file train.jsonl --no-validate bl dataset validate --file train.jsonl ``` +```bash +bl dataset validate --file dpo.jsonl --schema dpo +``` + ```bash bl dataset validate --file eval.jsonl --full-validate ``` diff --git a/skills/bailian-cli/reference/finetune.md b/skills/bailian-cli/reference/finetune.md index f4b3f83..5c90927 100644 --- a/skills/bailian-cli/reference/finetune.md +++ b/skills/bailian-cli/reference/finetune.md @@ -159,6 +159,9 @@ bl finetune checkpoints --job-id ft-xxx --output json - --datasets / --validations accept either file-ids (from `bl dataset - upload`) or local .jsonl paths. Local paths are validated and uploaded - first, then their file-ids are submitted — a one-step upload-and-train. +- Pre-submit gate: if the training dataset's sample count is not greater +- than batch_size, the job is rejected before upload or quota consumption +- (the platform would otherwise fail ~10 min in, after data processing). #### Examples