mirror of
https://github.com/modelstudioai/cli.git
synced 2026-09-14 19:49:23 +08:00
feat: refact validator to support dpo dataset
This commit is contained in:
故璃
@@ -3,6 +3,7 @@ import {
|
||||
detectOutputFormat,
|
||||
uploadDataset,
|
||||
validateDataset,
|
||||
parseDatasetSchemaFlag,
|
||||
MAX_DATASET_BYTES,
|
||||
BailianError,
|
||||
ExitCode,
|
||||
@@ -28,7 +29,8 @@ function formatIssue(issue: ValidationResult["errors"][number]): string {
|
||||
export default defineCommand({
|
||||
name: "dataset upload",
|
||||
description: "Upload a dataset file (.jsonl) to Bailian",
|
||||
usage: "bl dataset upload --file <path> [--purpose <name>] [--no-validate] [--full-validate]",
|
||||
usage:
|
||||
"bl dataset upload --file <path> [--purpose <name>] [--schema <chatml|dpo>] [--no-validate] [--full-validate]",
|
||||
options: [
|
||||
{
|
||||
flag: "--file <path>",
|
||||
@@ -39,6 +41,11 @@ export default defineCommand({
|
||||
flag: "--purpose <name>",
|
||||
description: 'Dataset purpose tag (default: "fine-tune"; e.g. "evaluation")',
|
||||
},
|
||||
{
|
||||
flag: "--schema <s>",
|
||||
description:
|
||||
'Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record.',
|
||||
},
|
||||
{
|
||||
flag: "--no-validate",
|
||||
description: "Skip the local JSONL pre-flight check (not recommended)",
|
||||
@@ -52,15 +59,19 @@ export default defineCommand({
|
||||
],
|
||||
examples: [
|
||||
"bl dataset upload --file train.jsonl",
|
||||
"bl dataset upload --file dpo.jsonl --schema dpo",
|
||||
"bl dataset upload --file eval.jsonl --purpose evaluation",
|
||||
"bl dataset upload --file train.jsonl --full-validate",
|
||||
"bl dataset upload --file train.jsonl --no-validate",
|
||||
],
|
||||
notes: [
|
||||
"Only .jsonl is supported in this release. The default validator expects a",
|
||||
'ChatML schema (each line a JSON object with a "messages" array). Other',
|
||||
"purposes may carry a different schema in the future and would be served",
|
||||
"by a purpose-specific validator at that point.",
|
||||
"Only .jsonl is supported in this release. Two record schemas are",
|
||||
"recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],",
|
||||
"chosen, rejected} where chosen/rejected are single assistant messages.",
|
||||
"With no --schema, a record carrying chosen/rejected is validated as DPO;",
|
||||
"pass --schema dpo to require it on every record, or --schema chatml to",
|
||||
"ignore preference fields. Other purposes may carry a different schema in",
|
||||
"the future and would be served by a purpose-specific validator.",
|
||||
"The dataset upload cap is 300MB per file.",
|
||||
"Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so",
|
||||
"the purpose tag is persisted (the DashScope-native /api/v1/files drops it).",
|
||||
@@ -72,10 +83,11 @@ export default defineCommand({
|
||||
const purpose = (flags.purpose as string | undefined) || "fine-tune";
|
||||
const skipValidate = Boolean(flags.noValidate);
|
||||
const fullValidate = Boolean(flags.fullValidate);
|
||||
const schema = parseDatasetSchemaFlag(flags.schema as string | undefined);
|
||||
const format = detectOutputFormat(config.output);
|
||||
|
||||
if (!skipValidate) {
|
||||
const result = await validateDataset(filePath!, { fullValidate });
|
||||
const result = await validateDataset(filePath!, { fullValidate, schema });
|
||||
if (!result.valid) {
|
||||
const lines = [
|
||||
`Dataset validation failed for ${filePath}`,
|
||||
@@ -112,6 +124,7 @@ export default defineCommand({
|
||||
purpose,
|
||||
max_bytes: MAX_DATASET_BYTES,
|
||||
validate: !skipValidate,
|
||||
schema: schema ?? "auto",
|
||||
},
|
||||
format,
|
||||
);
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
defineCommand,
|
||||
detectOutputFormat,
|
||||
validateDataset,
|
||||
parseDatasetSchemaFlag,
|
||||
BailianError,
|
||||
ExitCode,
|
||||
type Config,
|
||||
@@ -32,7 +33,7 @@ function formatStats(r: ValidationResult): string[] {
|
||||
export default defineCommand({
|
||||
name: "dataset validate",
|
||||
description: "Locally validate a dataset file (.jsonl) without uploading",
|
||||
usage: "bl dataset validate --file <path> [--full-validate]",
|
||||
usage: "bl dataset validate --file <path> [--full-validate] [--schema <chatml|dpo>]",
|
||||
options: [
|
||||
{ flag: "--file <path>", description: "Local .jsonl dataset file", required: true },
|
||||
{
|
||||
@@ -40,16 +41,26 @@ export default defineCommand({
|
||||
description: "JSON.parse every line instead of sampling (slower)",
|
||||
type: "boolean",
|
||||
},
|
||||
{
|
||||
flag: "--schema <s>",
|
||||
description:
|
||||
'Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record.',
|
||||
},
|
||||
],
|
||||
examples: [
|
||||
"bl dataset validate --file train.jsonl",
|
||||
"bl dataset validate --file dpo.jsonl --schema dpo",
|
||||
"bl dataset validate --file eval.jsonl --full-validate",
|
||||
"bl dataset validate --file train.jsonl --output json",
|
||||
],
|
||||
notes: [
|
||||
"Default scan: every line gets a structural check, then ~160 lines (front 50,",
|
||||
"evenly spaced 100, last 10) are JSON.parsed against the active schema.",
|
||||
"Today the only registered .jsonl schema is ChatML (messages array).",
|
||||
"Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,",
|
||||
"rejected} where chosen/rejected are single assistant messages. With no",
|
||||
"--schema, a record carrying chosen/rejected is validated as DPO; pass",
|
||||
"--schema dpo to require chosen/rejected on every record (strict), or",
|
||||
"--schema chatml to ignore preference fields.",
|
||||
"Use --full-validate to JSON.parse every line.",
|
||||
],
|
||||
async run(config: Config, flags: GlobalFlags) {
|
||||
@@ -57,14 +68,23 @@ export default defineCommand({
|
||||
if (!filePath) failIfMissing("file", "bl dataset validate --file <path>");
|
||||
|
||||
const fullValidate = Boolean(flags.fullValidate);
|
||||
const schema = parseDatasetSchemaFlag(flags.schema as string | undefined);
|
||||
const format = detectOutputFormat(config.output);
|
||||
|
||||
if (config.dryRun) {
|
||||
emitResult({ action: "dataset.validate", file: filePath, full: fullValidate }, format);
|
||||
emitResult(
|
||||
{
|
||||
action: "dataset.validate",
|
||||
file: filePath,
|
||||
full: fullValidate,
|
||||
schema: schema ?? "auto",
|
||||
},
|
||||
format,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await validateDataset(filePath!, { fullValidate });
|
||||
const result = await validateDataset(filePath!, { fullValidate, schema });
|
||||
|
||||
if (format === "json") {
|
||||
// For json output we always emit the structured result, exit code conveys validity.
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
validateDataset,
|
||||
fetchModelCapability,
|
||||
listSupportedTrainingTypes,
|
||||
preflightBatchSizeGate,
|
||||
isTrainingTypeCli,
|
||||
toServerTrainingType,
|
||||
TRAINING_TYPES_CLI,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
type CreateFineTuneRequest,
|
||||
type FineTuneHyperParameters,
|
||||
type DatasetFile,
|
||||
type DatasetSchema,
|
||||
type ValidationResult,
|
||||
} from "bailian-cli-core";
|
||||
import { existsSync, statSync } from "fs";
|
||||
@@ -50,30 +52,44 @@ function formatIssue(issue: ValidationResult["errors"][number]): string {
|
||||
}
|
||||
|
||||
interface ResolvedDataset {
|
||||
/** file-ids in input order (local uploads resolved to their new ids). */
|
||||
/**
|
||||
* Tokens in input order. Local paths are kept as-is here (a placeholder
|
||||
* until `uploadResolvedLocal` swaps them for real file-ids); bare file-ids
|
||||
* pass through untouched. In dry-run the paths stay (the previewed body
|
||||
* reflects exactly what the user typed).
|
||||
*/
|
||||
fileIds: string[];
|
||||
/** local paths that were uploaded (empty in dry-run). */
|
||||
uploaded: DatasetFile[];
|
||||
/** local paths recorded but not uploaded (dry-run only). */
|
||||
pendingPaths: string[];
|
||||
/** in-hand size for the first token, if known (avoids a redundant getDataset). */
|
||||
/** Local paths in input order, for the deferred upload step. */
|
||||
localPaths: string[];
|
||||
/** In-hand size for the first local token, if known (local statSync). */
|
||||
firstSize?: number;
|
||||
/**
|
||||
* Total training-sample count across local tokens, when known. Sourced from
|
||||
* `validateDataset`'s `stats.totalRecords` (summed per token). Undefined when
|
||||
* any token is a bare file-id (no local file to count) or in dry-run — the
|
||||
* pre-submit batch-size gate only fires when this is known, so file-id flows
|
||||
* fall through to the platform rather than risk a false positive.
|
||||
*/
|
||||
recordCount?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a comma-separated `--datasets` / `--validations` value into
|
||||
* file-ids, uploading any local paths through the same pipeline as
|
||||
* `bl dataset upload` (validate → upload). File-id tokens are passed through.
|
||||
* Analyze a comma-separated `--datasets` / `--validations` value WITHOUT
|
||||
* uploading: bare file-ids pass through; local paths are validated through the
|
||||
* same pipeline as `bl dataset upload` (so structural errors surface here),
|
||||
* their sample count and size are captured for the pre-submit gate, and the
|
||||
* path itself is recorded in `localPaths` for a later, deferred upload.
|
||||
*
|
||||
* In dry-run mode no upload happens: local paths are recorded in
|
||||
* `pendingPaths` and left in `fileIds` as-is so the previewed body still
|
||||
* reflects what the user typed.
|
||||
* Splitting analysis from upload lets the batch-size gate fire before any
|
||||
* network call — a doomed job (too few samples) is rejected without burning an
|
||||
* upload, and is offline-testable. In dry-run mode local paths are not
|
||||
* validated (the preview never touches the network or the disk beyond stat).
|
||||
*/
|
||||
async function resolveDatasetTokens(
|
||||
async function analyzeDatasetTokens(
|
||||
config: Config,
|
||||
raw: string,
|
||||
purpose: string,
|
||||
label: string,
|
||||
schema?: DatasetSchema,
|
||||
): Promise<ResolvedDataset> {
|
||||
const tokens = raw
|
||||
.split(",")
|
||||
@@ -84,23 +100,32 @@ async function resolveDatasetTokens(
|
||||
}
|
||||
|
||||
const fileIds: string[] = [];
|
||||
const uploaded: DatasetFile[] = [];
|
||||
const pendingPaths: string[] = [];
|
||||
const localPaths: string[] = [];
|
||||
let firstSize: number | undefined;
|
||||
let recordCount: number | undefined;
|
||||
// A file-id token has no local file to count, so the total sample count is
|
||||
// only knowable when every token is a local path. Once any file-id is seen,
|
||||
// flip to unknown and stop accumulating to avoid an undercount that could
|
||||
// trip the batch-size gate falsely.
|
||||
let recordCountKnown = true;
|
||||
|
||||
for (const [index, token] of tokens.entries()) {
|
||||
for (const token of tokens) {
|
||||
if (!isLocalPath(token)) {
|
||||
fileIds.push(token);
|
||||
continue;
|
||||
}
|
||||
if (config.dryRun) {
|
||||
pendingPaths.push(token);
|
||||
fileIds.push(token);
|
||||
recordCountKnown = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Local path → validate then upload (same flow as `bl dataset upload`).
|
||||
const result = await validateDataset(token);
|
||||
fileIds.push(token);
|
||||
localPaths.push(token);
|
||||
|
||||
if (config.dryRun) continue;
|
||||
|
||||
// Local path → validate (same checks as `bl dataset upload`). Upload is
|
||||
// deferred to `uploadResolvedLocal` so the gate can run first. The schema
|
||||
// (SFT vs DPO) is derived from --training-type so a DPO job validates the
|
||||
// chosen/rejected preference pairs here, not on the platform.
|
||||
const result = await validateDataset(token, { schema });
|
||||
if (!result.valid) {
|
||||
const lines = [
|
||||
`Dataset validation failed for ${token}`,
|
||||
@@ -129,6 +154,40 @@ async function resolveDatasetTokens(
|
||||
}
|
||||
}
|
||||
|
||||
// Accumulate the sample count so the caller can pre-flight the batch-size
|
||||
// gate before submitting. `totalRecords` is set by the jsonl validator as
|
||||
// (non-blank lines); undefined stats fall back to "unknown" (no gate).
|
||||
const tokenRecords = result.stats.totalRecords;
|
||||
if (typeof tokenRecords === "number") {
|
||||
recordCount = (recordCount ?? 0) + tokenRecords;
|
||||
}
|
||||
if (firstSize === undefined) firstSize = statSync(token).size;
|
||||
}
|
||||
|
||||
return {
|
||||
fileIds,
|
||||
localPaths,
|
||||
firstSize,
|
||||
recordCount: recordCountKnown ? recordCount : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload each local path recorded in `resolved.localPaths`, swapping the
|
||||
* placeholder path entries in `resolved.fileIds` for the returned file-ids.
|
||||
* Returns the uploaded file records (for the confirmation panel). No-op in
|
||||
* dry-run. Validation already happened in `analyzeDatasetTokens`, so this is
|
||||
* pure upload.
|
||||
*/
|
||||
async function uploadResolvedLocal(
|
||||
config: Config,
|
||||
resolved: ResolvedDataset,
|
||||
purpose: string,
|
||||
label: string,
|
||||
): Promise<DatasetFile[]> {
|
||||
const uploaded: DatasetFile[] = [];
|
||||
for (const [index, token] of resolved.fileIds.entries()) {
|
||||
if (!isLocalPath(token)) continue;
|
||||
const file: DatasetFile = await uploadDataset(config, { filePath: token, purpose });
|
||||
if (!file.file_id) {
|
||||
throw new BailianError(
|
||||
@@ -137,17 +196,14 @@ async function resolveDatasetTokens(
|
||||
);
|
||||
}
|
||||
uploaded.push(file);
|
||||
fileIds.push(file.file_id);
|
||||
if (index === 0) firstSize = file.size;
|
||||
|
||||
resolved.fileIds[index] = file.file_id;
|
||||
if (!config.quiet) {
|
||||
process.stderr.write(
|
||||
`Uploaded ${basename(token)} → ${file.file_id} (auto from --${label})\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { fileIds, uploaded, pendingPaths, firstSize };
|
||||
return uploaded;
|
||||
}
|
||||
|
||||
export default defineCommand({
|
||||
@@ -232,6 +288,9 @@ export default defineCommand({
|
||||
"--datasets / --validations accept either file-ids (from `bl dataset",
|
||||
"upload`) or local .jsonl paths. Local paths are validated and uploaded",
|
||||
"first, then their file-ids are submitted — a one-step upload-and-train.",
|
||||
"Pre-submit gate: if the training dataset's sample count is not greater",
|
||||
"than batch_size, the job is rejected before upload or quota consumption",
|
||||
"(the platform would otherwise fail ~10 min in, after data processing).",
|
||||
],
|
||||
async run(config: Config, flags: GlobalFlags) {
|
||||
const model = flags.model as string | undefined;
|
||||
@@ -240,18 +299,11 @@ export default defineCommand({
|
||||
const datasetsRaw = flags.datasets as string | undefined;
|
||||
if (!datasetsRaw) failIfMissing("datasets", "bl finetune create --datasets <ids|paths>");
|
||||
|
||||
const training = await resolveDatasetTokens(config, datasetsRaw!, "fine-tune", "datasets");
|
||||
const trainingFileIds = training.fileIds;
|
||||
|
||||
const validationsRaw = flags.validations as string | undefined;
|
||||
const validation = validationsRaw
|
||||
? await resolveDatasetTokens(config, validationsRaw, "fine-tune", "validations")
|
||||
: undefined;
|
||||
const validationFileIds = validation?.fileIds;
|
||||
|
||||
// Resolve the training type before analyzing datasets so the validator can
|
||||
// enforce the right record schema (DPO jobs require chosen/rejected on
|
||||
// every record). Whitelist is the single source of truth in core
|
||||
// (TRAINING_TYPES_CLI); any other value is rejected up-front.
|
||||
const trainingType = (flags.trainingType as string | undefined) || DEFAULT_TRAINING_TYPE;
|
||||
// Whitelist is the single source of truth in core (TRAINING_TYPES_CLI);
|
||||
// any other value is rejected up-front with an actionable error.
|
||||
if (!isTrainingTypeCli(trainingType)) {
|
||||
throw new BailianError(
|
||||
`--training-type "${trainingType}" is not supported.`,
|
||||
@@ -259,6 +311,18 @@ export default defineCommand({
|
||||
`Supported values: ${TRAINING_TYPES_CLI.join(", ")} (default: ${DEFAULT_TRAINING_TYPE}).`,
|
||||
);
|
||||
}
|
||||
// dpo / dpo-lora → "dpo" schema (strict chosen/rejected); else ChatML.
|
||||
const datasetSchema: DatasetSchema = trainingType.startsWith("dpo") ? "dpo" : "chatml";
|
||||
|
||||
const training = await analyzeDatasetTokens(config, datasetsRaw!, "datasets", datasetSchema);
|
||||
const trainingFileIds = training.fileIds;
|
||||
|
||||
const validationsRaw = flags.validations as string | undefined;
|
||||
const validation = validationsRaw
|
||||
? await analyzeDatasetTokens(config, validationsRaw, "validations", datasetSchema)
|
||||
: undefined;
|
||||
const validationFileIds = validation?.fileIds;
|
||||
|
||||
const modelName = flags.modelName as string | undefined;
|
||||
const suffix = flags.suffix as string | undefined;
|
||||
|
||||
@@ -299,6 +363,48 @@ export default defineCommand({
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-submit batch-size gate: the platform rejects a job whose number of
|
||||
// training samples is not greater than batch_size, but only surfaces that
|
||||
// ~10 minutes into the run (after data processing). Fail fast here, before
|
||||
// burning quota. `recordCount` is only known when every --datasets token
|
||||
// was a local file we validated; file-id tokens fall through to the
|
||||
// platform rather than risk a false positive from an undercount.
|
||||
//
|
||||
// The decision lives in core (`preflightBatchSizeGate`) — a structured,
|
||||
// job-level pre-flight that returns a `ValidationIssue` (same shape / stable
|
||||
// code as `validateDataset`) so the failure surfaces through the same
|
||||
// `BailianError` + issue convention used by `bl dataset upload`/`validate`.
|
||||
// ExitCode.GENERAL matches the existing validation-failed exit code.
|
||||
if (!config.dryRun && training.recordCount !== undefined) {
|
||||
// 16 is the platform default when neither the user nor the small-file
|
||||
// auto-adjust set a batch_size (see the auto-adjust comment above).
|
||||
const effectiveBatchSize = hp.batch_size ?? 16;
|
||||
const gate = preflightBatchSizeGate({
|
||||
recordCount: training.recordCount,
|
||||
batchSize: effectiveBatchSize,
|
||||
});
|
||||
if (!gate.ok && gate.issue) {
|
||||
throw new BailianError(gate.issue.message, ExitCode.GENERAL, gate.hint);
|
||||
}
|
||||
}
|
||||
|
||||
// Upload local paths now that the gate has cleared them. This swaps the
|
||||
// placeholder path entries in `training.fileIds` / `validation?.fileIds`
|
||||
// for real file-ids, so the body and confirmation panel below see ids.
|
||||
let uploadedTraining: DatasetFile[] = [];
|
||||
let uploadedValidation: DatasetFile[] = [];
|
||||
if (!config.dryRun) {
|
||||
uploadedTraining = await uploadResolvedLocal(config, training, "fine-tune", "datasets");
|
||||
if (validation) {
|
||||
uploadedValidation = await uploadResolvedLocal(
|
||||
config,
|
||||
validation,
|
||||
"fine-tune",
|
||||
"validations",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const body: CreateFineTuneRequest = {
|
||||
model: model!,
|
||||
training_file_ids: trainingFileIds,
|
||||
@@ -316,8 +422,8 @@ export default defineCommand({
|
||||
|
||||
if (config.dryRun) {
|
||||
const pending = [
|
||||
...training.pendingPaths.map((path) => ({ field: "datasets", path })),
|
||||
...(validation?.pendingPaths ?? []).map((path) => ({ field: "validations", path })),
|
||||
...training.localPaths.map((path) => ({ field: "datasets", path })),
|
||||
...(validation?.localPaths ?? []).map((path) => ({ field: "validations", path })),
|
||||
];
|
||||
emitResult(
|
||||
pending.length > 0
|
||||
@@ -353,10 +459,10 @@ export default defineCommand({
|
||||
if (validationFileIds) {
|
||||
process.stderr.write(` Validation: ${validationFileIds.join(", ")}\n`);
|
||||
}
|
||||
for (const file of training.uploaded) {
|
||||
for (const file of uploadedTraining) {
|
||||
process.stderr.write(` Uploaded: ${file.name} → ${file.file_id}\n`);
|
||||
}
|
||||
for (const file of validation?.uploaded ?? []) {
|
||||
for (const file of uploadedValidation) {
|
||||
process.stderr.write(` Uploaded: ${file.name} → ${file.file_id} (validation)\n`);
|
||||
}
|
||||
process.stderr.write(` n_epochs: ${hp.n_epochs}\n`);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}}
|
||||
@@ -0,0 +1,2 @@
|
||||
{"messages":[{"role":"user","content":"你能帮我写一篇文章吗?"}],"chosen":{"role":"assistant","content":"当然可以，请告诉我具体方向。"},"rejected":{"role":"assistant","content":"可以。"}}
|
||||
{"messages":[{"role":"user","content":"安排一下明天的日程?"}],"chosen":{"role":"assistant","content":"当然，请告诉我具体事项。"},"rejected":{"role":"assistant","content":"好的。"}}
|
||||
@@ -79,6 +79,98 @@ describe("e2e: dataset (offline)", () => {
|
||||
expect(data.action).toBe("dataset.upload");
|
||||
expect(data.validate).toBe(false);
|
||||
});
|
||||
|
||||
test("dataset validate 自动识别 DPO 并校验 chosen/rejected", async () => {
|
||||
// No --schema: a record carrying chosen/rejected is auto-detected as DPO
|
||||
// and the valid fixture passes.
|
||||
const file = join(__dirname, ".dataset-dpo-valid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"dataset",
|
||||
"validate",
|
||||
"--file",
|
||||
file,
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode, stderr).toBe(0);
|
||||
const data = parseStdoutJson<{ valid: boolean; stats: { totalRecords?: number } }>(stdout);
|
||||
expect(data.valid).toBe(true);
|
||||
expect(data.stats.totalRecords).toBe(2);
|
||||
});
|
||||
|
||||
test("dataset validate --schema dpo 拒绝缺失 rejected 的记录", async () => {
|
||||
const file = join(__dirname, ".dataset-dpo-invalid.jsonl");
|
||||
const { stdout, exitCode } = await runCli([
|
||||
"dataset",
|
||||
"validate",
|
||||
"--file",
|
||||
file,
|
||||
"--schema",
|
||||
"dpo",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode).not.toBe(0);
|
||||
const data = parseStdoutJson<{ valid: boolean; errors: { code: string; path?: string }[] }>(
|
||||
stdout,
|
||||
);
|
||||
expect(data.valid).toBe(false);
|
||||
expect(data.errors.map((e) => e.code)).toContain("MISSING_REJECTED");
|
||||
});
|
||||
|
||||
test("dataset validate --schema chatml 忽略 chosen/rejected（不报 DPO 错误）", async () => {
|
||||
// Same invalid-DPO file, but --schema chatml must not run DPO checks.
|
||||
const file = join(__dirname, ".dataset-dpo-invalid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"dataset",
|
||||
"validate",
|
||||
"--file",
|
||||
file,
|
||||
"--schema",
|
||||
"chatml",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode, stderr).toBe(0);
|
||||
const data = parseStdoutJson<{ valid: boolean; errors: { code: string }[] }>(stdout);
|
||||
expect(data.valid).toBe(true);
|
||||
expect(data.errors.filter((c) => c.code.startsWith("MISSING_"))).toEqual([]);
|
||||
});
|
||||
|
||||
test("dataset validate --schema <bad> 以非零码退出", async () => {
|
||||
const file = join(__dirname, ".dataset-valid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"dataset",
|
||||
"validate",
|
||||
"--file",
|
||||
file,
|
||||
"--schema",
|
||||
"sft",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode).not.toBe(0);
|
||||
expect(`${stdout}\n${stderr}`).toMatch(/Unsupported --schema/);
|
||||
});
|
||||
|
||||
test("dataset upload --dry-run 转发 --schema", async () => {
|
||||
const file = join(__dirname, ".dataset-dpo-valid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"dataset",
|
||||
"upload",
|
||||
"--file",
|
||||
file,
|
||||
"--schema",
|
||||
"dpo",
|
||||
"--dry-run",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode, stderr).toBe(0);
|
||||
const data = parseStdoutJson<{ action: string; schema: string }>(stdout);
|
||||
expect(data.action).toBe("dataset.upload");
|
||||
expect(data.schema).toBe("dpo");
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isDashScopeE2EReady())("e2e: dataset (DashScope)", () => {
|
||||
|
||||
@@ -177,6 +177,51 @@ describe("e2e: finetune (offline)", () => {
|
||||
expect(exitCode, stdout + stderr).not.toBe(0);
|
||||
});
|
||||
|
||||
test("finetune create 样本数 <= batch_size 时提交前快速失败且不上传", async () => {
|
||||
// The fixture has 3 records; the small-file auto-adjust sets batch_size=8,
|
||||
// so 3 <= 8 trips the pre-submit gate. The gate fires before any upload,
|
||||
// so this is fully offline (no key, no network) — the proof is that the
|
||||
// error is the gate message AND no "Uploaded …" line ever appears.
|
||||
const localPath = join(cliPackageRoot, "tests", "e2e", ".dataset-valid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"finetune",
|
||||
"create",
|
||||
"--model",
|
||||
"qwen3-8b",
|
||||
"--datasets",
|
||||
localPath,
|
||||
"--yes",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode, stdout + stderr).not.toBe(0);
|
||||
const combined = `${stdout}\n${stderr}`;
|
||||
expect(combined).toMatch(/not greater than batch_size/i);
|
||||
// Crucially, no upload happened — the gate must fire before the upload step.
|
||||
expect(combined).not.toMatch(/Uploaded .* → file-/);
|
||||
});
|
||||
|
||||
test("finetune create --batch-size 过小仍按 8 下限比较（不绕过卡口）", async () => {
|
||||
// Even with --batch-size 1 (server clamps to 8), 3 samples <= 8 still trips
|
||||
// the gate — confirms the gate uses the clamped/effective batch, not the raw.
|
||||
const localPath = join(cliPackageRoot, "tests", "e2e", ".dataset-valid.jsonl");
|
||||
const { stdout, stderr, exitCode } = await runCli([
|
||||
"finetune",
|
||||
"create",
|
||||
"--model",
|
||||
"qwen3-8b",
|
||||
"--datasets",
|
||||
localPath,
|
||||
"--batch-size",
|
||||
"1",
|
||||
"--yes",
|
||||
"--output",
|
||||
"json",
|
||||
]);
|
||||
expect(exitCode, stdout + stderr).not.toBe(0);
|
||||
expect(`${stdout}\n${stderr}`).toMatch(/batch_size \(8\)/);
|
||||
});
|
||||
|
||||
test.each([
|
||||
["list", ["--status", "RUNNING"]],
|
||||
["get", ["--job-id", "ft-xxx"]],
|
||||
|
||||
@@ -6,10 +6,12 @@ export {
|
||||
registerValidator,
|
||||
listSupportedFormats,
|
||||
MAX_DATASET_BYTES,
|
||||
parseDatasetSchemaFlag,
|
||||
} from "./validate/index.ts";
|
||||
export type {
|
||||
ValidatorSpec,
|
||||
ValidateOpts,
|
||||
DatasetSchema,
|
||||
ValidationResult,
|
||||
ValidationIssue,
|
||||
ValidationSeverity,
|
||||
|
||||
@@ -9,7 +9,7 @@ import { existsSync, statSync } from "fs";
|
||||
import { extname } from "path";
|
||||
import { BailianError } from "../../errors/base.ts";
|
||||
import { ExitCode } from "../../errors/codes.ts";
|
||||
import type { ValidationIssue, ValidationStats } from "./types.ts";
|
||||
import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts";
|
||||
|
||||
/**
|
||||
* The platform caps dataset uploads at 300MB per file. `bl dataset upload`
|
||||
@@ -66,6 +66,23 @@ export function emptyStats(): ValidationStats {
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `--schema` CLI value into a `DatasetSchema` (or `undefined` for
|
||||
* auto-detect). Single source of truth for the schema vocabulary so `dataset
|
||||
* validate`, `dataset upload`, and any future caller agree on accepted values
|
||||
* and error wording. Throws USAGE for anything unrecognized.
|
||||
*/
|
||||
export function parseDatasetSchemaFlag(value: string | undefined): DatasetSchema | undefined {
|
||||
if (value === undefined || value.trim() === "") return undefined;
|
||||
const v = value.trim();
|
||||
if (v === "chatml" || v === "dpo") return v;
|
||||
throw new BailianError(
|
||||
`Unsupported --schema "${value}". Supported: chatml, dpo.`,
|
||||
ExitCode.USAGE,
|
||||
`Omit --schema to auto-detect per record (a record with chosen/rejected is treated as DPO).`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Produce a deterministic set of sample line indices for deep checking.
|
||||
* Indices are 1-based to match what users see in editors / error messages.
|
||||
*
|
||||
|
||||
@@ -4,10 +4,11 @@ export {
|
||||
registerValidator,
|
||||
listSupportedFormats,
|
||||
} from "./registry.ts";
|
||||
export { MAX_DATASET_BYTES } from "./common.ts";
|
||||
export { MAX_DATASET_BYTES, parseDatasetSchemaFlag } from "./common.ts";
|
||||
export type {
|
||||
ValidatorSpec,
|
||||
ValidateOpts,
|
||||
DatasetSchema,
|
||||
ValidationResult,
|
||||
ValidationIssue,
|
||||
ValidationSeverity,
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
/**
|
||||
* JSONL validator for ChatML-style datasets (e.g. SFT training data).
|
||||
*
|
||||
* Schema scope: each line is `{"messages": [{role, content}, ...]}` with
|
||||
* roles in (system, user, assistant). This matches the platform's documented
|
||||
* SFT training format. Other JSONL schemas (e.g. evaluation datasets with
|
||||
* different field shapes) should ship their own validator and register it —
|
||||
* the registry can be extended in the future to dispatch on `(extension,
|
||||
* purpose)` rather than extension alone if a purpose-specific .jsonl schema
|
||||
* appears.
|
||||
* Schema scope: the `.jsonl` ChatML family. Two record shapes are recognized:
|
||||
* - SFT: `{"messages": [{role, content}, ...]}`
|
||||
* - DPO: `{"messages": [...], "chosen": {role, content}, "rejected": {...}}`
|
||||
* `chosen`/`rejected` are single assistant messages — the preferred vs
|
||||
* dispreferred response. Which shape is enforced is selected by
|
||||
* `ValidateOpts.schema` (`"chatml"` | `"dpo"`), defaulting to per-record
|
||||
* auto-detect. Other JSONL schemas (e.g. evaluation datasets with a different
|
||||
* field shape) should ship their own validator and register it — the registry
|
||||
* can be extended in the future to dispatch on `(extension, purpose)` rather
|
||||
* than extension alone if a purpose-specific .jsonl schema appears.
|
||||
*
|
||||
* Two-stage strategy (see decision log):
|
||||
* 1. Quick scan — readline pass over the entire file checking only that
|
||||
@@ -21,7 +24,13 @@
|
||||
*/
|
||||
import { createReadStream } from "fs";
|
||||
import { createInterface } from "readline";
|
||||
import type { ValidatorSpec, ValidateOpts, ValidationResult, ValidationIssue } from "./types.ts";
|
||||
import type {
|
||||
ValidatorSpec,
|
||||
ValidateOpts,
|
||||
ValidationResult,
|
||||
ValidationIssue,
|
||||
DatasetSchema,
|
||||
} from "./types.ts";
|
||||
import { makeIssue, pickSampleLines } from "./common.ts";
|
||||
|
||||
const VALID_ROLES = new Set(["system", "user", "assistant"]);
|
||||
@@ -76,6 +85,7 @@ async function deepCheck(
|
||||
filePath: string,
|
||||
totalLines: number,
|
||||
fullValidate: boolean,
|
||||
schema: DatasetSchema | undefined,
|
||||
signal?: AbortSignal,
|
||||
): Promise<DeepCheckResult> {
|
||||
const targetSet = fullValidate ? null : new Set(pickSampleLines(totalLines));
|
||||
@@ -108,30 +118,98 @@ async function deepCheck(
|
||||
continue;
|
||||
}
|
||||
|
||||
issues.push(...inspectChatMLRecord(obj, lineNo));
|
||||
issues.push(...inspectRecord(obj, lineNo, schema));
|
||||
}
|
||||
return { sampled, issues };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate one ChatML record. Hard errors are returned with severity "error",
|
||||
* advisory checks (role ordering) as "warning". Caller dedupes/aggregates.
|
||||
* Structural checks for a single message object `{role, content}`. Shared by
|
||||
* the `messages[]` entries and the DPO `chosen` / `rejected` preference fields
|
||||
* (which are each a single assistant message). Caller-supplied `path` scopes
|
||||
* the issue location (e.g. `messages[2]` vs `chosen`).
|
||||
*/
|
||||
function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] {
|
||||
function inspectMessageObject(msg: unknown, lineNo: number, path: string): ValidationIssue[] {
|
||||
const out: ValidationIssue[] = [];
|
||||
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) {
|
||||
if (msg === null || typeof msg !== "object" || Array.isArray(msg)) {
|
||||
out.push(
|
||||
makeIssue("error", "MESSAGE_NOT_OBJECT", `Message must be an object.`, {
|
||||
line: lineNo,
|
||||
path,
|
||||
}),
|
||||
);
|
||||
return out;
|
||||
}
|
||||
const m = msg as Record<string, unknown>;
|
||||
const role = m.role;
|
||||
const content = m.content;
|
||||
if (typeof role !== "string" || !VALID_ROLES.has(role)) {
|
||||
out.push(
|
||||
makeIssue(
|
||||
"error",
|
||||
"INVALID_ROLE",
|
||||
`Invalid role "${String(role)}". Expected one of: system, user, assistant.`,
|
||||
{ line: lineNo, path: `${path}.role` },
|
||||
),
|
||||
);
|
||||
}
|
||||
if (typeof content !== "string") {
|
||||
out.push(
|
||||
makeIssue("error", "INVALID_CONTENT", `"content" must be a string (got ${typeof content}).`, {
|
||||
line: lineNo,
|
||||
path: `${path}.content`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch one record to the right schema inspector.
|
||||
*
|
||||
* SFT and DPO are not sibling schemas — DPO is a *superset* of SFT
|
||||
* (`{messages:[...], chosen, rejected}` = the ChatML prompt + a preference
|
||||
* pair). So this dispatcher only decides *whether* to also validate the
|
||||
* preference fields; the `messages[]` core is always handled by
|
||||
* `inspectChatMLRecord` (DPO calls into it).
|
||||
*
|
||||
* Schema selection mirrors the `ValidateOpts.schema` contract:
|
||||
* - `"chatml"` → SFT only (preference fields ignored).
|
||||
* - `"dpo"` → DPO, strictly (every record must carry chosen+rejected).
|
||||
* - `undefined` (auto) → per record: DPO when `chosen`/`rejected` present, else SFT.
|
||||
*/
|
||||
function inspectRecord(obj: unknown, lineNo: number, schema?: DatasetSchema): ValidationIssue[] {
|
||||
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) {
|
||||
return [
|
||||
makeIssue(
|
||||
"error",
|
||||
"RECORD_NOT_OBJECT",
|
||||
`Each line must be a JSON object, got ${Array.isArray(obj) ? "array" : typeof obj}.`,
|
||||
{ line: lineNo },
|
||||
),
|
||||
);
|
||||
return out;
|
||||
];
|
||||
}
|
||||
|
||||
const record = obj as Record<string, unknown>;
|
||||
const hasChosen = "chosen" in record;
|
||||
const hasRejected = "rejected" in record;
|
||||
const isDpo = schema === "dpo" || (schema === undefined && (hasChosen || hasRejected));
|
||||
return isDpo
|
||||
? inspectDPORecord(record, lineNo, hasChosen, hasRejected)
|
||||
: inspectChatMLRecord(record, lineNo);
|
||||
}
|
||||
|
||||
/**
|
||||
* SFT (ChatML) record: `{"messages": [{role, content}, ...]}`.
|
||||
*
|
||||
* Validates the shared `messages[]` core that every ChatML-family record
|
||||
* carries — including DPO, which is why `inspectDPORecord` delegates here for
|
||||
* the prompt portion. `chosen`/`rejected`, if present on the record, are
|
||||
* intentionally ignored: callers wanting those checked must go through DPO
|
||||
* mode. Hard errors return as "error", advisory role-ordering checks as
|
||||
* "warning".
|
||||
*/
|
||||
function inspectChatMLRecord(record: Record<string, unknown>, lineNo: number): ValidationIssue[] {
|
||||
const out: ValidationIssue[] = [];
|
||||
const messages = record.messages;
|
||||
if (!Array.isArray(messages)) {
|
||||
out.push(
|
||||
@@ -159,38 +237,8 @@ function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] {
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const msg = messages[i];
|
||||
const path = `messages[${i}]`;
|
||||
if (msg === null || typeof msg !== "object" || Array.isArray(msg)) {
|
||||
out.push(
|
||||
makeIssue("error", "MESSAGE_NOT_OBJECT", `Message must be an object.`, {
|
||||
line: lineNo,
|
||||
path,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const m = msg as Record<string, unknown>;
|
||||
const role = m.role;
|
||||
const content = m.content;
|
||||
if (typeof role !== "string" || !VALID_ROLES.has(role)) {
|
||||
out.push(
|
||||
makeIssue(
|
||||
"error",
|
||||
"INVALID_ROLE",
|
||||
`Invalid role "${String(role)}". Expected one of: system, user, assistant.`,
|
||||
{ line: lineNo, path: `${path}.role` },
|
||||
),
|
||||
);
|
||||
}
|
||||
if (typeof content !== "string") {
|
||||
out.push(
|
||||
makeIssue(
|
||||
"error",
|
||||
"INVALID_CONTENT",
|
||||
`"content" must be a string (got ${typeof content}).`,
|
||||
{ line: lineNo, path: `${path}.content` },
|
||||
),
|
||||
);
|
||||
}
|
||||
out.push(...inspectMessageObject(msg, lineNo, path));
|
||||
const role = (msg as Record<string, unknown> | null)?.role;
|
||||
|
||||
if (role === "system") {
|
||||
if (i !== 0) {
|
||||
@@ -238,6 +286,76 @@ function inspectChatMLRecord(obj: unknown, lineNo: number): ValidationIssue[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* DPO record: `{"messages": [...], "chosen": {role, content}, "rejected": {...}}`.
|
||||
*
|
||||
* The prompt context (`messages[]`) is validated by `inspectChatMLRecord`;
|
||||
* this function adds the preference pair on top. `chosen`/`rejected` are each a
|
||||
* single assistant message — the preferred vs dispreferred response — so they
|
||||
* reuse `inspectMessageObject` with a scoped `path`.
|
||||
*
|
||||
* If the prompt is structurally broken (missing/empty `messages`), the SFT
|
||||
* inspector already reported the hard error and we skip preference checks — a
|
||||
* record missing its prompt is too broken to meaningfully check chosen/rejected
|
||||
* on top, matching the original early-return semantics.
|
||||
*/
|
||||
function inspectDPORecord(
|
||||
record: Record<string, unknown>,
|
||||
lineNo: number,
|
||||
hasChosen: boolean,
|
||||
hasRejected: boolean,
|
||||
): ValidationIssue[] {
|
||||
const out = inspectChatMLRecord(record, lineNo);
|
||||
const messages = record.messages;
|
||||
if (!Array.isArray(messages) || messages.length === 0) return out;
|
||||
|
||||
if (!hasChosen) {
|
||||
out.push(
|
||||
makeIssue("error", "MISSING_CHOSEN", `DPO record is missing the "chosen" preference.`, {
|
||||
line: lineNo,
|
||||
path: "chosen",
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (!hasRejected) {
|
||||
out.push(
|
||||
makeIssue("error", "MISSING_REJECTED", `DPO record is missing the "rejected" preference.`, {
|
||||
line: lineNo,
|
||||
path: "rejected",
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (hasChosen) {
|
||||
out.push(...inspectMessageObject(record.chosen, lineNo, "chosen"));
|
||||
const role = (record.chosen as Record<string, unknown> | null)?.role;
|
||||
if (typeof role === "string" && role !== "assistant") {
|
||||
out.push(
|
||||
makeIssue(
|
||||
"warning",
|
||||
"PREFERENCE_ROLE_NOT_ASSISTANT",
|
||||
`"chosen" role should be "assistant" (got "${role}").`,
|
||||
{ line: lineNo, path: "chosen.role" },
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (hasRejected) {
|
||||
out.push(...inspectMessageObject(record.rejected, lineNo, "rejected"));
|
||||
const role = (record.rejected as Record<string, unknown> | null)?.role;
|
||||
if (typeof role === "string" && role !== "assistant") {
|
||||
out.push(
|
||||
makeIssue(
|
||||
"warning",
|
||||
"PREFERENCE_ROLE_NOT_ASSISTANT",
|
||||
`"rejected" role should be "assistant" (got "${role}").`,
|
||||
{ line: lineNo, path: "rejected.role" },
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export const jsonlValidator: ValidatorSpec = {
|
||||
format: "jsonl",
|
||||
extensions: [".jsonl"],
|
||||
@@ -280,6 +398,7 @@ export const jsonlValidator: ValidatorSpec = {
|
||||
filePath,
|
||||
quick.totalLines,
|
||||
Boolean(opts.fullValidate),
|
||||
opts.schema,
|
||||
opts.signal,
|
||||
);
|
||||
|
||||
|
||||
@@ -16,8 +16,24 @@ export interface ValidateOpts {
|
||||
maxBytes?: number;
|
||||
/** Optional abort signal for long-running scans. */
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Record-schema selector for formats that carry more than one schema under
|
||||
* the same extension. Today only the `.jsonl` ChatML family honors it:
|
||||
* - `"chatml"` — `{messages: [...]}` (SFT). `chosen`/`rejected` ignored.
|
||||
* - `"dpo"` — `{messages: [...], chosen: {role,content}, rejected: {...}}`.
|
||||
* Every record MUST carry `chosen` + `rejected`.
|
||||
* - `undefined` — auto-detect per record: a record with `chosen` or
|
||||
* `rejected` is validated as DPO, otherwise as ChatML.
|
||||
* `finetune create` sets this from `--training-type` (dpo* → "dpo") so a DPO
|
||||
* job with malformed preference pairs fails at validate time, not on the
|
||||
* platform ten minutes in.
|
||||
*/
|
||||
schema?: DatasetSchema;
|
||||
}
|
||||
|
||||
/** The schemas a `.jsonl` record can be validated against. */
|
||||
export type DatasetSchema = "chatml" | "dpo";
|
||||
|
||||
export type ValidationSeverity = "error" | "warning";
|
||||
|
||||
export interface ValidationIssue {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export * from "./types.ts";
|
||||
export * from "./api.ts";
|
||||
export * from "./capability.ts";
|
||||
export * from "./preflight.ts";
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Finetune job-level pre-flight checks.
|
||||
*
|
||||
* Sibling to `capability.ts` (the model-capability pre-flight). These checks
|
||||
* are NOT dataset-format validations — they consume the per-file validation
|
||||
* output (e.g. `stats.totalRecords` from `validateDataset`) together with
|
||||
* job-level inputs (hyper-parameters) and decide whether a job is submittable.
|
||||
* Format/structure checks live in `dataset/validate/`; these live here because
|
||||
* they depend on concerns the format validators must never know about.
|
||||
*
|
||||
* Consistency with the validate architecture: a failing check returns a
|
||||
* `ValidationIssue` (same shape, stable `code`, `error` severity) so callers
|
||||
* surface it through the same `BailianError` + issue-list convention used by
|
||||
* `bl dataset upload` / `bl dataset validate`. The trigger stays inline in
|
||||
* `finetune create` (the only call site today) — that's the job-level boundary.
|
||||
*/
|
||||
import type { ValidationIssue } from "../dataset/validate/types.ts";
|
||||
|
||||
/** Stable issue code for "too few training samples for the batch size". */
|
||||
export const INSUFFICIENT_SAMPLES_CODE = "INSUFFICIENT_SAMPLES";
|
||||
|
||||
export interface BatchSizeGateInput {
|
||||
/**
|
||||
* Total training-sample count across all `--datasets` files. Sourced from
|
||||
* `validateDataset`'s `stats.totalRecords` (summed per file). The gate only
|
||||
* fires when this is known — i.e. every dataset token was a local file that
|
||||
* was validated; bare file-id tokens yield no count and fall through to the
|
||||
* platform.
|
||||
*/
|
||||
recordCount: number;
|
||||
/**
|
||||
* Effective batch_size the job will run with — after the CLI's clamp
|
||||
* ([8, 1024]) and small-file auto-adjust, or the platform default (16) when
|
||||
* neither the user nor auto-adjust set one.
|
||||
*/
|
||||
batchSize: number;
|
||||
}
|
||||
|
||||
export interface BatchSizeGateResult {
|
||||
ok: boolean;
|
||||
/** Present when `!ok`, in the same shape `validateDataset` issues use. */
|
||||
issue?: ValidationIssue;
|
||||
/** Actionable guidance; callers surface it as the `BailianError` detail. */
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-flight the platform's "training samples must exceed batch_size" rule.
|
||||
*
|
||||
* The platform rejects a job whose number of training samples is not greater
|
||||
* than batch_size, but only surfaces that ~10 minutes into the run (after data
|
||||
* processing). This gate fails fast, before upload or quota consumption.
|
||||
*
|
||||
* Conservative by design — never false-positives: with the platform's default
|
||||
* 0.9 train split, training samples = 0.9 * recordCount <= recordCount, so
|
||||
* `recordCount <= batchSize` implies training samples <= batchSize implies
|
||||
* certain platform failure. Borderline counts (records just above batchSize)
|
||||
* may still fail on the platform; that's an acceptable false negative for a
|
||||
* pre-check, and the hint nudges users to leave margin for the split.
|
||||
*/
|
||||
export function preflightBatchSizeGate(input: BatchSizeGateInput): BatchSizeGateResult {
|
||||
const { recordCount, batchSize } = input;
|
||||
if (recordCount > batchSize) return { ok: true };
|
||||
return {
|
||||
ok: false,
|
||||
issue: {
|
||||
severity: "error",
|
||||
code: INSUFFICIENT_SAMPLES_CODE,
|
||||
message: `Training dataset has ${recordCount} sample(s), which is not greater than batch_size (${batchSize}).`,
|
||||
},
|
||||
hint: [
|
||||
"The platform requires the number of training samples to exceed batch_size.",
|
||||
"Options:",
|
||||
" • add more data (recommended: comfortably more than batch_size, since the",
|
||||
" platform also holds back a default 0.9 train split),",
|
||||
" • lower --batch-size (server clamps to a minimum of 8).",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { afterAll, describe, expect, test } from "vite-plus/test";
|
||||
import { mkdirSync, rmSync, writeFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
import { tmpdir } from "os";
|
||||
import { validateDataset, parseDatasetSchemaFlag } from "../src/index.ts";
|
||||
|
||||
const tmp = join(tmpdir(), `bl-dpo-test-${process.pid}`);
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
|
||||
function file(name: string, lines: string[]): string {
|
||||
const p = join(tmp, name);
|
||||
writeFileSync(p, lines.join("\n"));
|
||||
return p;
|
||||
}
|
||||
|
||||
const DPO_OK =
|
||||
'{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}';
|
||||
const SFT_OK =
|
||||
'{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}]}';
|
||||
|
||||
afterAll(() => rmSync(tmp, { recursive: true, force: true }));
|
||||
|
||||
function codes(r: { errors: { code: string }[]; warnings: { code: string }[] }) {
|
||||
return {
|
||||
errors: r.errors.map((e) => e.code),
|
||||
warnings: r.warnings.map((w) => w.code),
|
||||
};
|
||||
}
|
||||
|
||||
describe("validateDataset — DPO schema", () => {
|
||||
test("valid DPO record passes under auto-detect and --schema dpo", async () => {
|
||||
const p = file("ok.jsonl", [DPO_OK]);
|
||||
const auto = await validateDataset(p, { fullValidate: true });
|
||||
expect(auto.valid).toBe(true);
|
||||
const dpo = await validateDataset(p, { fullValidate: true, schema: "dpo" });
|
||||
expect(dpo.valid).toBe(true);
|
||||
});
|
||||
|
||||
test("missing rejected → MISSING_REJECTED (auto-detect, since chosen present)", async () => {
|
||||
const p = file("miss_rej.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true });
|
||||
expect(r.valid).toBe(false);
|
||||
expect(codes(r).errors).toContain("MISSING_REJECTED");
|
||||
expect(codes(r).errors).not.toContain("MISSING_CHOSEN");
|
||||
});
|
||||
|
||||
test("missing chosen → MISSING_CHOSEN (auto-detect, since rejected present)", async () => {
|
||||
const p = file("miss_chosen.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"hi"}],"rejected":{"role":"assistant","content":"bad"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true });
|
||||
expect(r.valid).toBe(false);
|
||||
expect(codes(r).errors).toContain("MISSING_CHOSEN");
|
||||
});
|
||||
|
||||
test('schema "dpo" requires both chosen and rejected on every record', async () => {
|
||||
// A record with neither chosen nor rejected is SFT-shaped; under --schema dpo
|
||||
// it must be flagged as missing both preferences.
|
||||
const p = file("sft_under_dpo.jsonl", [SFT_OK]);
|
||||
const r = await validateDataset(p, { fullValidate: true, schema: "dpo" });
|
||||
expect(r.valid).toBe(false);
|
||||
expect(codes(r).errors).toEqual(expect.arrayContaining(["MISSING_CHOSEN", "MISSING_REJECTED"]));
|
||||
});
|
||||
|
||||
test('schema "chatml" ignores chosen/rejected (no DPO errors)', async () => {
|
||||
const p = file("miss_rej_chatml.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true, schema: "chatml" });
|
||||
expect(r.valid).toBe(true);
|
||||
expect(codes(r).errors.filter((c) => c.startsWith("MISSING_"))).toEqual([]);
|
||||
});
|
||||
|
||||
test("SFT-only file under auto-detect is unaffected (no DPO checks)", async () => {
|
||||
const p = file("sft.jsonl", [SFT_OK]);
|
||||
const r = await validateDataset(p, { fullValidate: true });
|
||||
expect(r.valid).toBe(true);
|
||||
expect(codes(r).errors).toEqual([]);
|
||||
});
|
||||
|
||||
test("chosen not a message object → MESSAGE_NOT_OBJECT at path chosen", async () => {
|
||||
const p = file("bad_chosen.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"hi"}],"chosen":"nope","rejected":{"role":"assistant","content":"bad"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true });
|
||||
expect(r.valid).toBe(false);
|
||||
const err = r.errors.find((e) => e.code === "MESSAGE_NOT_OBJECT");
|
||||
expect(err).toBeDefined();
|
||||
expect(err!.path).toBe("chosen");
|
||||
});
|
||||
|
||||
test("chosen role=user → PREFERENCE_ROLE_NOT_ASSISTANT warning", async () => {
|
||||
const p = file("role_warn.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"user","content":"good"},"rejected":{"role":"assistant","content":"bad"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true });
|
||||
expect(r.valid).toBe(true);
|
||||
expect(codes(r).warnings).toContain("PREFERENCE_ROLE_NOT_ASSISTANT");
|
||||
});
|
||||
|
||||
test("multi-turn prompt in messages still validates with DPO preferences", async () => {
|
||||
const p = file("multiturn.jsonl", [
|
||||
'{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"},{"role":"user","content":"c"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}',
|
||||
]);
|
||||
const r = await validateDataset(p, { fullValidate: true, schema: "dpo" });
|
||||
expect(r.valid).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseDatasetSchemaFlag", () => {
|
||||
test("undefined / empty → undefined (auto)", () => {
|
||||
expect(parseDatasetSchemaFlag(undefined)).toBeUndefined();
|
||||
expect(parseDatasetSchemaFlag("")).toBeUndefined();
|
||||
expect(parseDatasetSchemaFlag(" ")).toBeUndefined();
|
||||
});
|
||||
|
||||
test("chatml / dpo pass through", () => {
|
||||
expect(parseDatasetSchemaFlag("chatml")).toBe("chatml");
|
||||
expect(parseDatasetSchemaFlag("dpo")).toBe("dpo");
|
||||
expect(parseDatasetSchemaFlag(" dpo ")).toBe("dpo");
|
||||
});
|
||||
|
||||
test("unrecognized throws", () => {
|
||||
expect(() => parseDatasetSchemaFlag("sft")).toThrow(/Unsupported --schema/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, test } from "vite-plus/test";
|
||||
import { preflightBatchSizeGate, INSUFFICIENT_SAMPLES_CODE } from "../src/index.ts";
|
||||
|
||||
describe("preflightBatchSizeGate", () => {
|
||||
test("passes when recordCount exceeds batch_size", () => {
|
||||
const r = preflightBatchSizeGate({ recordCount: 9, batchSize: 8 });
|
||||
expect(r.ok).toBe(true);
|
||||
expect(r.issue).toBeUndefined();
|
||||
expect(r.hint).toBeUndefined();
|
||||
});
|
||||
|
||||
test("passes at the boundary just above batch_size (9 > 8)", () => {
|
||||
expect(preflightBatchSizeGate({ recordCount: 9, batchSize: 8 }).ok).toBe(true);
|
||||
// A comfortably-large dataset is fine too.
|
||||
expect(preflightBatchSizeGate({ recordCount: 1000, batchSize: 16 }).ok).toBe(true);
|
||||
});
|
||||
|
||||
test("fails when recordCount equals batch_size (must be *greater than*)", () => {
|
||||
const r = preflightBatchSizeGate({ recordCount: 8, batchSize: 8 });
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.issue).toBeDefined();
|
||||
expect(r.issue!.severity).toBe("error");
|
||||
expect(r.issue!.code).toBe(INSUFFICIENT_SAMPLES_CODE);
|
||||
expect(r.issue!.message).toMatch(/not greater than batch_size \(8\)/);
|
||||
expect(r.hint).toMatch(/add more data/);
|
||||
});
|
||||
|
||||
test("fails when recordCount is below batch_size (the 3-sample / batch-8 case)", () => {
|
||||
const r = preflightBatchSizeGate({ recordCount: 3, batchSize: 8 });
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.issue!.message).toMatch(/3 sample\(s\)/);
|
||||
expect(r.issue!.message).toMatch(/batch_size \(8\)/);
|
||||
expect(r.hint).toMatch(/lower --batch-size/);
|
||||
});
|
||||
|
||||
test("hint references the 0.9 train split so users leave margin", () => {
|
||||
const r = preflightBatchSizeGate({ recordCount: 5, batchSize: 8 });
|
||||
expect(r.hint).toMatch(/0\.9 train split/);
|
||||
});
|
||||
|
||||
test("honors the effective (clamped) batch size, not a raw sub-minimum", () => {
|
||||
// The CLI clamps --batch-size 1 up to 8 before calling; 3 <= 8 still fails.
|
||||
const r = preflightBatchSizeGate({ recordCount: 3, batchSize: 8 });
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.issue!.message).toMatch(/batch_size \(8\)/);
|
||||
});
|
||||
});
|
||||
@@ -102,27 +102,31 @@ bl dataset list --output json
|
||||
|
||||
### `bl dataset upload`
|
||||
|
||||
| Field | Value |
|
||||
| --------------- | -------------------------------------------------------------------------------------- |
|
||||
| **Name** | `dataset upload` |
|
||||
| **Description** | Upload a dataset file (.jsonl) to Bailian |
|
||||
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--no-validate] [--full-validate]` |
|
||||
| Field | Value |
|
||||
| --------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| **Name** | `dataset upload` |
|
||||
| **Description** | Upload a dataset file (.jsonl) to Bailian |
|
||||
| **Usage** | `bl dataset upload --file <path> [--purpose <name>] [--schema <chatml\|dpo>] [--no-validate] [--full-validate]` |
|
||||
|
||||
#### Options
|
||||
|
||||
| Flag | Type | Required | Description |
|
||||
| ------------------ | ------- | -------- | ------------------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local .jsonl dataset file (≤300MB) |
|
||||
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
||||
| `--no-validate` | boolean | no | Skip the local JSONL pre-flight check (not recommended) |
|
||||
| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) |
|
||||
| Flag | Type | Required | Description |
|
||||
| ------------------ | ------- | -------- | --------------------------------------------------------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local .jsonl dataset file (≤300MB) |
|
||||
| `--purpose <name>` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record. |
|
||||
| `--no-validate` | boolean | no | Skip the local JSONL pre-flight check (not recommended) |
|
||||
| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) |
|
||||
|
||||
#### Notes
|
||||
|
||||
- Only .jsonl is supported in this release. The default validator expects a
|
||||
- ChatML schema (each line a JSON object with a "messages" array). Other
|
||||
- purposes may carry a different schema in the future and would be served
|
||||
- by a purpose-specific validator at that point.
|
||||
- Only .jsonl is supported in this release. Two record schemas are
|
||||
- recognized: chatml = {messages:[...]} (SFT); dpo = {messages:[...],
|
||||
- chosen, rejected} where chosen/rejected are single assistant messages.
|
||||
- With no --schema, a record carrying chosen/rejected is validated as DPO;
|
||||
- pass --schema dpo to require it on every record, or --schema chatml to
|
||||
- ignore preference fields. Other purposes may carry a different schema in
|
||||
- the future and would be served by a purpose-specific validator.
|
||||
- The dataset upload cap is 300MB per file.
|
||||
- Upload uses the OpenAI-compatible /compatible-mode/v1/files endpoint so
|
||||
- the purpose tag is persisted (the DashScope-native /api/v1/files drops it).
|
||||
@@ -133,6 +137,10 @@ bl dataset list --output json
|
||||
bl dataset upload --file train.jsonl
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset upload --file dpo.jsonl --schema dpo
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset upload --file eval.jsonl --purpose evaluation
|
||||
```
|
||||
@@ -147,24 +155,29 @@ bl dataset upload --file train.jsonl --no-validate
|
||||
|
||||
### `bl dataset validate`
|
||||
|
||||
| Field | Value |
|
||||
| --------------- | ---------------------------------------------------------- |
|
||||
| **Name** | `dataset validate` |
|
||||
| **Description** | Locally validate a dataset file (.jsonl) without uploading |
|
||||
| **Usage** | `bl dataset validate --file <path> [--full-validate]` |
|
||||
| Field | Value |
|
||||
| --------------- | ------------------------------------------------------------------------------ |
|
||||
| **Name** | `dataset validate` |
|
||||
| **Description** | Locally validate a dataset file (.jsonl) without uploading |
|
||||
| **Usage** | `bl dataset validate --file <path> [--full-validate] [--schema <chatml\|dpo>]` |
|
||||
|
||||
#### Options
|
||||
|
||||
| Flag | Type | Required | Description |
|
||||
| ----------------- | ------- | -------- | -------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local .jsonl dataset file |
|
||||
| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) |
|
||||
| Flag | Type | Required | Description |
|
||||
| ----------------- | ------- | -------- | --------------------------------------------------------------------------------------------------- |
|
||||
| `--file <path>` | string | yes | Local .jsonl dataset file |
|
||||
| `--full-validate` | boolean | no | JSON.parse every line instead of sampling (slower) |
|
||||
| `--schema <s>` | string | no | Record schema: "chatml" (SFT) or "dpo" (requires chosen/rejected). Default auto-detects per record. |
|
||||
|
||||
#### Notes
|
||||
|
||||
- Default scan: every line gets a structural check, then ~160 lines (front 50,
|
||||
- evenly spaced 100, last 10) are JSON.parsed against the active schema.
|
||||
- Today the only registered .jsonl schema is ChatML (messages array).
|
||||
- Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,
|
||||
- rejected} where chosen/rejected are single assistant messages. With no
|
||||
- --schema, a record carrying chosen/rejected is validated as DPO; pass
|
||||
- --schema dpo to require chosen/rejected on every record (strict), or
|
||||
- --schema chatml to ignore preference fields.
|
||||
- Use --full-validate to JSON.parse every line.
|
||||
|
||||
#### Examples
|
||||
@@ -173,6 +186,10 @@ bl dataset upload --file train.jsonl --no-validate
|
||||
bl dataset validate --file train.jsonl
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset validate --file dpo.jsonl --schema dpo
|
||||
```
|
||||
|
||||
```bash
|
||||
bl dataset validate --file eval.jsonl --full-validate
|
||||
```
|
||||
|
||||
@@ -159,6 +159,9 @@ bl finetune checkpoints --job-id ft-xxx --output json
|
||||
- --datasets / --validations accept either file-ids (from `bl dataset
|
||||
- upload`) or local .jsonl paths. Local paths are validated and uploaded
|
||||
- first, then their file-ids are submitted — a one-step upload-and-train.
|
||||
- Pre-submit gate: if the training dataset's sample count is not greater
|
||||
- than batch_size, the job is rejected before upload or quota consumption
|
||||
- (the platform would otherwise fail ~10 min in, after data processing).
|
||||
|
||||
#### Examples
|
||||
|
||||
|
||||
Reference in New Issue
Block a user