diff --git a/packages/commands/src/commands/dataset/upload.ts b/packages/commands/src/commands/dataset/upload.ts index 6888b11..822c5a9 100644 --- a/packages/commands/src/commands/dataset/upload.ts +++ b/packages/commands/src/commands/dataset/upload.ts @@ -5,6 +5,7 @@ import { parseDatasetSchemaFlag, formatIssue, MAX_DATASET_BYTES, + MAX_CPT_BYTES, MAX_MEDIA_ZIP_BYTES, BailianError, ExitCode, @@ -16,7 +17,7 @@ const UPLOAD_FLAGS = { file: { type: "string", valueHint: "", - description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image)", + description: "Local dataset file (.jsonl or .zip; ≤200MB SFT/DPO, ≤300MB CPT, ≤2GB media zip)", required: true, }, purpose: { @@ -57,13 +58,14 @@ export default defineCommand({ ], notes: [ "Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl", - "manifest). Five record schemas are recognized: chatml = {messages:[...]}", + "manifest). Six record schemas are recognized: chatml = {messages:[...]}", '(SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."}', '(continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav",', 'text:"..."} (audio fine-tuning); image = {img_path:"..."} (image', - "generation). With no --schema, a record carrying wav_fn is validated as", - "TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT,", - "otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the", + "generation); video = {first_frame_path:...} (video generation). With no", + "--schema, a record carrying wav_fn is validated as TTS, img_path as image,", + "chosen/rejected as DPO, text (no messages) as CPT, otherwise ChatML.", + "Upload cap: 200MB SFT/DPO text, 300MB CPT, 2GB media zip. Upload uses the", "OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is", "persisted (the DashScope-native /api/v1/files drops it).", ], @@ -72,11 +74,15 @@ export default defineCommand({ const filePath = flags.file; const purpose = flags.purpose || "fine-tune"; const schema = parseDatasetSchemaFlag(flags.schema); - // Image and video schemas allow larger ZIPs (1 GB vs 300 MB for text). + // Size caps differ per training type: SFT/DPO 200MB, CPT 300MB, media ZIP 2GB. const isMediaSchema = schema === "image" || schema === "video"; + const maxBytes = isMediaSchema + ? MAX_MEDIA_ZIP_BYTES + : schema === "cpt" + ? MAX_CPT_BYTES + : MAX_DATASET_BYTES; if (!flags.noValidate) { - const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES; const result = await validateDataset(filePath, { fullValidate: flags.fullValidate, schema, @@ -116,7 +122,7 @@ export default defineCommand({ action: "dataset.upload", file: filePath, purpose, - max_bytes: isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES, + max_bytes: maxBytes, validate: !flags.noValidate, schema: schema ?? "auto", }, diff --git a/packages/core/src/dataset/index.ts b/packages/core/src/dataset/index.ts index cc605c4..b6e581c 100644 --- a/packages/core/src/dataset/index.ts +++ b/packages/core/src/dataset/index.ts @@ -7,6 +7,7 @@ export { registerValidator, listSupportedFormats, MAX_DATASET_BYTES, + MAX_CPT_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag, formatIssue, diff --git a/packages/core/src/dataset/validate/common.ts b/packages/core/src/dataset/validate/common.ts index 17bdbad..212fe7b 100644 --- a/packages/core/src/dataset/validate/common.ts +++ b/packages/core/src/dataset/validate/common.ts @@ -12,18 +12,24 @@ import { ExitCode } from "../../errors/codes.ts"; import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts"; /** - * The platform caps dataset uploads at 300MB per file. `bl dataset upload` - * enforces this client-side so users learn early. Update if the platform - * raises the cap or differentiates per-purpose limits. + * The platform caps SFT/DPO text dataset uploads at 200MB per file. + * `bl dataset upload` enforces this client-side so users learn early. + * CPT uses 300MB (see MAX_CPT_BYTES); API general upload is also 300MB. */ -export const MAX_DATASET_BYTES = 300 * 1024 * 1024; +export const MAX_DATASET_BYTES = 200 * 1024 * 1024; /** - * Image / video ZIP size cap — 1 GB per the platform docs (vs 300 MB for - * text / audio). Used by `bl dataset upload` for media schemas and by the - * `sft-lora` training profile for image / video validation. + * CPT text dataset size cap — 300 MB per the platform docs. + * CPT requires at least 50M tokens; larger files are expected. */ -export const MAX_MEDIA_ZIP_BYTES = 1024 * 1024 * 1024; +export const MAX_CPT_BYTES = 300 * 1024 * 1024; + +/** + * Image / video ZIP size cap — 2 GB per the platform docs. Used by + * `bl dataset upload` for media schemas and by the `sft-lora` training + * profile for image / video validation. + */ +export const MAX_MEDIA_ZIP_BYTES = 2 * 1024 * 1024 * 1024; export interface PreflightResult { bytes: number; diff --git a/packages/core/src/dataset/validate/index.ts b/packages/core/src/dataset/validate/index.ts index 42df442..169a32c 100644 --- a/packages/core/src/dataset/validate/index.ts +++ b/packages/core/src/dataset/validate/index.ts @@ -4,7 +4,12 @@ export { registerValidator, listSupportedFormats, } from "./registry.ts"; -export { MAX_DATASET_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag } from "./common.ts"; +export { + MAX_DATASET_BYTES, + MAX_CPT_BYTES, + MAX_MEDIA_ZIP_BYTES, + parseDatasetSchemaFlag, +} from "./common.ts"; export { formatIssue } from "./format.ts"; export type { ValidatorSpec, diff --git a/packages/core/src/dataset/validate/schemas/chatml.ts b/packages/core/src/dataset/validate/schemas/chatml.ts index d4c032e..959149e 100644 --- a/packages/core/src/dataset/validate/schemas/chatml.ts +++ b/packages/core/src/dataset/validate/schemas/chatml.ts @@ -5,12 +5,215 @@ * and no more specific schema matches, ChatML is selected. `inspectMessageObject` * lives here because it is the canonical per-message check; the DPO schema * imports it to validate `chosen` / `rejected` preference messages. + * + * Content format: supports both legacy plain-string content (`"content": "…"`) + * and the current platform array format (`"content": [{"text": "…"}, …]`). + * The array format may also carry `image` / `video` items for VL multimodal + * understanding data. + * + * Tool calling: supports `role: "tool"` messages with `tool_call_id`, and + * `assistant.tool_calls` arrays. Validates id correspondence. */ import { makeIssue } from "../common.ts"; import type { ValidationIssue } from "../types.ts"; import type { RecordSchemaSpec } from "./types.ts"; -const VALID_ROLES = new Set(["system", "user", "assistant"]); +const VALID_ROLES = new Set(["system", "user", "assistant", "tool"]); + +/** + * Validate a content field that may be: + * - A plain string (legacy format) + * - An array of content items: `[{text: "…"}, {image: "…"}, {video: "…"|"…"}, …]` + * + * Returns issues found. `path` scopes the location for error reporting. + */ +export function inspectContentField( + content: unknown, + lineNo: number, + path: string, +): ValidationIssue[] { + const out: ValidationIssue[] = []; + if (typeof content === "string") { + // Legacy string format — always valid. + return out; + } + if (!Array.isArray(content)) { + out.push( + makeIssue( + "error", + "INVALID_CONTENT", + `"content" must be a string or an array of content items (got ${typeof content}).`, + { line: lineNo, path }, + ), + ); + return out; + } + if (content.length === 0) { + out.push( + makeIssue("error", "EMPTY_CONTENT_ARRAY", `"content" array must not be empty.`, { + line: lineNo, + path, + }), + ); + return out; + } + for (let idx = 0; idx < content.length; idx++) { + const item = content[idx]; + const itemPath = `${path}[${idx}]`; + if (item === null || typeof item !== "object" || Array.isArray(item)) { + out.push( + makeIssue("error", "INVALID_CONTENT_ITEM", `Content item must be an object.`, { + line: lineNo, + path: itemPath, + }), + ); + continue; + } + const obj = item as Record; + const hasText = "text" in obj; + const hasImage = "image" in obj; + const hasVideo = "video" in obj; + if (!hasText && !hasImage && !hasVideo) { + out.push( + makeIssue( + "error", + "CONTENT_ITEM_NO_KNOWN_FIELD", + `Content item must contain at least one of: "text", "image", "video".`, + { line: lineNo, path: itemPath }, + ), + ); + continue; + } + if (hasText && typeof obj.text !== "string") { + out.push( + makeIssue("error", "INVALID_CONTENT_TEXT", `"text" in content item must be a string.`, { + line: lineNo, + path: `${itemPath}.text`, + }), + ); + } + if (hasImage && typeof obj.image !== "string") { + out.push( + makeIssue("error", "INVALID_CONTENT_IMAGE", `"image" in content item must be a string.`, { + line: lineNo, + path: `${itemPath}.image`, + }), + ); + } + if (hasVideo) { + // video can be a string (file path) or an array of strings (frame list) + const video = obj.video; + if (typeof video !== "string" && !Array.isArray(video)) { + out.push( + makeIssue( + "error", + "INVALID_CONTENT_VIDEO", + `"video" in content item must be a string (file path) or an array of strings (frame list).`, + { line: lineNo, path: `${itemPath}.video` }, + ), + ); + } else if (Array.isArray(video)) { + for (let frameIdx = 0; frameIdx < video.length; frameIdx++) { + if (typeof video[frameIdx] !== "string") { + out.push( + makeIssue( + "error", + "INVALID_VIDEO_FRAME", + `Video frame list item at index ${frameIdx} must be a string.`, + { line: lineNo, path: `${itemPath}.video[${frameIdx}]` }, + ), + ); + } + } + } + } + } + return out; +} + +/** + * Validate `tool_calls` array on an assistant message. + * Each entry: `{id: string, type: "function", function: {name: string, arguments: string}}`. + */ +function inspectToolCalls(toolCalls: unknown, lineNo: number, path: string): ValidationIssue[] { + const out: ValidationIssue[] = []; + if (!Array.isArray(toolCalls)) { + out.push( + makeIssue("error", "INVALID_TOOL_CALLS", `"tool_calls" must be an array.`, { + line: lineNo, + path, + }), + ); + return out; + } + for (let idx = 0; idx < toolCalls.length; idx++) { + const call = toolCalls[idx]; + const callPath = `${path}[${idx}]`; + if (call === null || typeof call !== "object" || Array.isArray(call)) { + out.push( + makeIssue("error", "INVALID_TOOL_CALL", `tool_calls item must be an object.`, { + line: lineNo, + path: callPath, + }), + ); + continue; + } + const obj = call as Record; + if (typeof obj.id !== "string" || obj.id.length === 0) { + out.push( + makeIssue("error", "TOOL_CALL_MISSING_ID", `tool_calls item must have a non-empty "id".`, { + line: lineNo, + path: `${callPath}.id`, + }), + ); + } + if (obj.type !== "function") { + out.push( + makeIssue( + "warning", + "TOOL_CALL_TYPE_NOT_FUNCTION", + `tool_calls item "type" should be "function" (got "${String(obj.type)}").`, + { line: lineNo, path: `${callPath}.type` }, + ), + ); + } + const fn = obj.function; + if (fn === null || typeof fn !== "object" || Array.isArray(fn)) { + out.push( + makeIssue( + "error", + "TOOL_CALL_MISSING_FUNCTION", + `tool_calls item must have a "function" object.`, + { + line: lineNo, + path: `${callPath}.function`, + }, + ), + ); + } else { + const fnObj = fn as Record; + if (typeof fnObj.name !== "string" || fnObj.name.length === 0) { + out.push( + makeIssue("error", "TOOL_CALL_FN_NO_NAME", `tool_calls function must have a "name".`, { + line: lineNo, + path: `${callPath}.function.name`, + }), + ); + } + if (typeof fnObj.arguments !== "string") { + out.push( + makeIssue( + "error", + "TOOL_CALL_FN_ARGS_NOT_STRING", + `tool_calls function "arguments" must be a JSON string.`, + { line: lineNo, path: `${callPath}.function.arguments` }, + ), + ); + } + } + } + return out; +} /** * Structural checks for a single message object `{role, content}`. Shared by @@ -35,25 +238,73 @@ export function inspectMessageObject( } const record = msg as Record; const role = record.role; - const content = record.content; if (typeof role !== "string" || !VALID_ROLES.has(role)) { out.push( makeIssue( "error", "INVALID_ROLE", - `Invalid role "${String(role)}". Expected one of: system, user, assistant.`, + `Invalid role "${String(role)}". Expected one of: system, user, assistant, tool.`, { line: lineNo, path: `${path}.role` }, ), ); } - if (typeof content !== "string") { + + // tool role: must have tool_call_id + if (role === "tool") { + if (typeof record.tool_call_id !== "string" || record.tool_call_id.length === 0) { + out.push( + makeIssue( + "error", + "TOOL_MISSING_CALL_ID", + `A "tool" role message must have a non-empty "tool_call_id".`, + { line: lineNo, path: `${path}.tool_call_id` }, + ), + ); + } + } + + // content validation: string or array format + if (!("content" in record)) { + // assistant messages with tool_calls may omit content + if (role !== "assistant" || !("tool_calls" in record)) { + out.push( + makeIssue("error", "MISSING_CONTENT", `"content" field is missing.`, { + line: lineNo, + path: `${path}.content`, + }), + ); + } + } else { + out.push(...inspectContentField(record.content, lineNo, `${path}.content`)); + } + + // tool_calls on assistant + if ("tool_calls" in record) { + out.push(...inspectToolCalls(record.tool_calls, lineNo, `${path}.tool_calls`)); + } + + // OpenAI migration guard: name / weight are not supported by Bailian + if ("name" in record) { out.push( - makeIssue("error", "INVALID_CONTENT", `"content" must be a string (got ${typeof content}).`, { - line: lineNo, - path: `${path}.content`, - }), + makeIssue( + "warning", + "UNSUPPORTED_FIELD_NAME", + `Field "name" is not supported by Bailian. Remove it when migrating from OpenAI/Azure.`, + { line: lineNo, path: `${path}.name` }, + ), ); } + if ("weight" in record) { + out.push( + makeIssue( + "warning", + "UNSUPPORTED_FIELD_WEIGHT", + `Field "weight" is not supported by Bailian. Remove it when migrating from OpenAI/Azure.`, + { line: lineNo, path: `${path}.weight` }, + ), + ); + } + return out; } @@ -91,19 +342,24 @@ export function inspectChatMLRecord( let sawSystem = false; let lastRole: string | undefined; - for (let i = 0; i < messages.length; i++) { - const msg = messages[i]; - const path = `messages[${i}]`; + let lastAssistantIdx = -1; + const toolCallIds = new Set(); + const toolResponseIds = new Set(); + + for (let idx = 0; idx < messages.length; idx++) { + const msg = messages[idx]; + const path = `messages[${idx}]`; out.push(...inspectMessageObject(msg, lineNo, path)); - const role = (msg as Record | null)?.role; + const msgObj = msg as Record | null; + const role = msgObj?.role; if (role === "system") { - if (i !== 0) { + if (idx !== 0) { out.push( makeIssue( "warning", "SYSTEM_NOT_FIRST", - `"system" message should appear at index 0; found at index ${i}.`, + `"system" message should appear at index 0; found at index ${idx}.`, { line: lineNo, path: `${path}.role` }, ), ); @@ -111,6 +367,27 @@ export function inspectChatMLRecord( sawSystem = true; } + if (role === "assistant") { + lastAssistantIdx = idx; + // Collect tool_calls ids + if (msgObj && Array.isArray(msgObj.tool_calls)) { + for (const call of msgObj.tool_calls) { + const callObj = call as Record | null; + if (callObj && typeof callObj.id === "string") { + toolCallIds.add(callObj.id); + } + } + } + } + + if (role === "tool") { + const callId = msgObj?.tool_call_id; + if (typeof callId === "string" && callId.length > 0) { + toolResponseIds.add(callId); + } + } + + // Consecutive same-role warning (skip tool — multiple tool responses are normal) if (lastRole === role && (role === "user" || role === "assistant")) { out.push( makeIssue( @@ -123,6 +400,7 @@ export function inspectChatMLRecord( } if (typeof role === "string") lastRole = role; } + // Soft check: messages without any user role almost certainly indicate a bug. if (!messages.some((m) => (m as Record).role === "user")) { out.push( @@ -140,9 +418,101 @@ export function inspectChatMLRecord( }), ); } + + // tool_call_id correspondence: every tool response should reference a known call id + for (const responseId of toolResponseIds) { + if (!toolCallIds.has(responseId)) { + out.push( + makeIssue( + "warning", + "TOOL_CALL_ID_UNMATCHED", + `tool message references tool_call_id "${responseId}" which does not match any assistant tool_calls[].id.`, + { line: lineNo, path: "messages" }, + ), + ); + } + } + + // thinking tag check: …` should only appear in the last assistant message + if (lastAssistantIdx >= 0) { + for (let idx = 0; idx < messages.length; idx++) { + if (idx === lastAssistantIdx) continue; + const msg = messages[idx] as Record | null; + if (msg?.role !== "assistant") continue; + const content = msg?.content; + if (contentHasThinkTag(content)) { + out.push( + makeIssue( + "warning", + "THINK_TAG_NOT_LAST", + `Thinking tags (…) should only appear in the last assistant message, found at messages[${idx}].`, + { line: lineNo, path: `messages[${idx}].content` }, + ), + ); + } + } + } + + // loss_weight validation (record-level, invite-only parameter) + if ("loss_weight" in record) { + const lossWeight = record.loss_weight; + if (typeof lossWeight !== "number" || lossWeight < 0 || lossWeight > 1) { + out.push( + makeIssue( + "error", + "INVALID_LOSS_WEIGHT", + `"loss_weight" must be a number between 0.0 and 1.0 (got ${JSON.stringify(lossWeight)}).`, + { line: lineNo, path: "loss_weight" }, + ), + ); + } + } + + // OpenAI migration guard at record level + if ("name" in record && !("messages" in record)) { + // Only warn at record level if it's not inside messages (messages handled above) + out.push( + makeIssue( + "warning", + "UNSUPPORTED_FIELD_NAME", + `Record-level field "name" is not supported by Bailian.`, + { line: lineNo, path: "name" }, + ), + ); + } + if ("weight" in record) { + out.push( + makeIssue( + "warning", + "UNSUPPORTED_FIELD_WEIGHT", + `Record-level field "weight" is not supported by Bailian. Remove it when migrating from OpenAI/Azure.`, + { line: lineNo, path: "weight" }, + ), + ); + } + return out; } +/** Check whether content (string or array) contains a tag. */ +function contentHasThinkTag(content: unknown): boolean { + if (typeof content === "string") { + return content.includes(""); + } + if (Array.isArray(content)) { + return content.some((item) => { + if (item && typeof item === "object" && "text" in item) { + return ( + typeof (item as Record).text === "string" && + ((item as Record).text as string).includes("") + ); + } + return false; + }); + } + return false; +} + /** * ChatML / SFT schema. The auto-detect predicate is `true` so it acts as the * registry fallback — any record that isn't picked up by a more specific diff --git a/packages/core/src/dataset/validate/schemas/dpo.ts b/packages/core/src/dataset/validate/schemas/dpo.ts index 5dc6937..26c3bcc 100644 --- a/packages/core/src/dataset/validate/schemas/dpo.ts +++ b/packages/core/src/dataset/validate/schemas/dpo.ts @@ -18,6 +18,22 @@ function inspectDPORecord(record: Record, lineNo: number): Vali const messages = record.messages; if (!Array.isArray(messages) || messages.length === 0) return out; + // DPO convention: messages should end with a user message (the prompt that + // chosen/rejected respond to). If the last message is assistant, it's likely + // a structural mistake. + const lastMsg = messages[messages.length - 1] as Record | null; + if (lastMsg && lastMsg.role !== "user") { + out.push( + makeIssue( + "warning", + "DPO_LAST_MSG_NOT_USER", + `DPO "messages" should end with a "user" message (the prompt for chosen/rejected). ` + + `Got "${String(lastMsg.role)}" as the last message.`, + { line: lineNo, path: `messages[${messages.length - 1}].role` }, + ), + ); + } + const hasChosen = "chosen" in record; const hasRejected = "rejected" in record; diff --git a/packages/core/src/dataset/validate/schemas/image.ts b/packages/core/src/dataset/validate/schemas/image.ts index daef380..248619c 100644 --- a/packages/core/src/dataset/validate/schemas/image.ts +++ b/packages/core/src/dataset/validate/schemas/image.ts @@ -16,7 +16,15 @@ import type { ValidationIssue } from "../types.ts"; import type { RecordSchemaSpec } from "./types.ts"; /** Accepted image file extensions (lower-case, with dot). */ -export const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tiff"]); +export const IMAGE_EXTENSIONS = new Set([ + ".png", + ".jpg", + ".jpeg", + ".bmp", + ".tif", + ".tiff", + ".webp", +]); /** * Check that a path string ends with an accepted image extension. diff --git a/packages/core/src/dataset/validate/zip.ts b/packages/core/src/dataset/validate/zip.ts index 4309133..da1c362 100644 --- a/packages/core/src/dataset/validate/zip.ts +++ b/packages/core/src/dataset/validate/zip.ts @@ -3,14 +3,15 @@ * * A training data ZIP must have: * - `data.jsonl` at the root — the manifest mapping media files to labels. - * - A `train/` subfolder (or media files at the root) referenced by the - * manifest entries. + * The platform requires data.jsonl to be directly visible when opening the + * ZIP (no wrapping folder). + * - Media files referenced by the manifest entries. * * This validator owns the **ZIP-level structural checks** (entries present, - * references resolve). The **per-record JSONL content validation** is delegated - * to the existing `jsonlValidator` — we extract `data.jsonl` to a temp file, - * run the full pipeline (quickScan + deepCheck + schema dispatch), and stitch - * the results together. + * references resolve, filename constraints). The **per-record JSONL content + * validation** is delegated to the existing `jsonlValidator` — we extract + * `data.jsonl` to a temp file, run the full pipeline (quickScan + deepCheck + + * schema dispatch), and stitch the results together. * * The schema for `data.jsonl` records is passed via `opts.schema` (typically * `"tts"` for audio). The profile layer decides which schema to use based on @@ -92,6 +93,121 @@ function collectZipEntries(zipPath: string): Promise { }); } +/** + * Platform filename constraints: + * - Allowed charset: ASCII letters (a-z, A-Z), digits (0-9), underscore (_), hyphen (-) + * - Filename (without extension) ≤ 120 characters + * - Filenames must be globally unique (ignoring extension) + */ +const FILENAME_CHARSET_RE = /^[a-zA-Z0-9_-]+$/; +const MAX_FILENAME_BASE_LENGTH = 120; + +/** Extract the base name (no extension) from a path segment. */ +function basenameNoExt(segment: string): string { + const dot = segment.lastIndexOf("."); + return dot > 0 ? segment.slice(0, dot) : segment; +} + +/** + * Validate ZIP entry filenames against platform constraints. + * Returns issues for charset violations, over-length names, and duplicates. + */ +function validateZipFilenames(entries: string[]): ValidationIssue[] { + const out: ValidationIssue[] = []; + const seenBasenames = new Map(); // basename (no ext) → first full path + const MAX_REPORTED = 10; + + for (const entry of entries) { + // Skip directory entries + if (entry.endsWith("/")) continue; + + // Check each path segment (folder names + file name) + const segments = entry.split("/").filter((s) => s.length > 0); + for (const segment of segments) { + // Strip extension for the charset check on the base part + const base = basenameNoExt(segment); + const ext = segment.slice(base.length); // includes dot, e.g. ".jpg" + + // Charset check on base name (extension checked separately) + if (base.length > 0 && !FILENAME_CHARSET_RE.test(base)) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "INVALID_FILENAME_CHARSET", + `File/folder name "${segment}" contains invalid characters. ` + + `Only a-z, A-Z, 0-9, underscore (_), and hyphen (-) are allowed.`, + { path: entry }, + ), + ); + } + } + + // Extension charset (allow dot + alphanumeric) + if (ext.length > 0 && !/^\.[a-zA-Z0-9]+$/.test(ext)) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "INVALID_FILENAME_CHARSET", + `File extension "${ext}" in "${segment}" contains invalid characters.`, + { path: entry }, + ), + ); + } + } + } + + // Filename length check (base name without extension) + const fileName = segments[segments.length - 1] ?? ""; + const baseName = basenameNoExt(fileName); + if (baseName.length > MAX_FILENAME_BASE_LENGTH) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "FILENAME_TOO_LONG", + `Filename "${fileName}" (without extension) exceeds ${MAX_FILENAME_BASE_LENGTH} characters ` + + `(got ${baseName.length}). Shorten the name and re-upload.`, + { path: entry }, + ), + ); + } + } + + // Global uniqueness check (ignoring extension, case-sensitive) + if (baseName.length > 0) { + const existing = seenBasenames.get(baseName); + if (existing !== undefined) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "DUPLICATE_FILENAME", + `Filename "${fileName}" conflicts with "${existing}" — names must be globally unique ` + + `(ignoring extension) even across different folders.`, + { path: entry }, + ), + ); + } + } else { + seenBasenames.set(baseName, entry); + } + } + } + + if (out.length >= MAX_REPORTED) { + out.push( + makeIssue( + "warning", + "FILENAME_ISSUES_TRUNCATED", + `More filename issues exist but reporting is capped at ${MAX_REPORTED}.`, + ), + ); + } + return out; +} + /** * Extract a single entry from a ZIP archive to a destination path. */ @@ -201,19 +317,37 @@ export const zipValidator: ValidatorSpec = { }; } - // --- 2. Check for data.jsonl --- - const hasDataJsonl = entries.some( - (entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"), - ); - if (!hasDataJsonl) { - errors.push( - makeIssue( - "error", - "MISSING_DATA_JSONL", - `ZIP archive must contain "data.jsonl" at the root. ` + - `This file maps media files (e.g. .wav) to their labels.`, - ), - ); + // --- 2. Check for data.jsonl (must be at ZIP root) --- + const hasRootDataJsonl = entries.some((entry) => entry === "data.jsonl"); + const nestedDataJsonl = + !hasRootDataJsonl && entries.find((entry) => entry.endsWith("/data.jsonl")); + if (!hasRootDataJsonl) { + if (nestedDataJsonl) { + errors.push( + makeIssue( + "error", + "DATA_JSONL_NOT_AT_ROOT", + `"data.jsonl" must be at the ZIP root (found "${nestedDataJsonl}"). ` + + `Re-package so that opening the ZIP shows data.jsonl directly, without a wrapping folder.`, + ), + ); + } else { + errors.push( + makeIssue( + "error", + "MISSING_DATA_JSONL", + `ZIP archive must contain "data.jsonl" at the root. ` + + `This file maps media files (e.g. .wav, .jpg) to their labels.`, + ), + ); + } + } + + // --- 2b. Filename constraints (charset, length, uniqueness) --- + const filenameIssues = validateZipFilenames(entries); + for (const issue of filenameIssues) { + if (issue.severity === "error") errors.push(issue); + else warnings.push(issue); } // --- 3. Check for train/ directory (modality-aware) --- @@ -265,8 +399,8 @@ export const zipValidator: ValidatorSpec = { } } - // If data.jsonl is missing, we can't do JSONL content validation. - if (!hasDataJsonl) { + // If data.jsonl is missing entirely, we can't do JSONL content validation. + if (!hasRootDataJsonl && !nestedDataJsonl) { return { valid: false, format: "zip", @@ -278,9 +412,11 @@ export const zipValidator: ValidatorSpec = { } // --- 4. Extract data.jsonl to a temp file and run jsonlValidator --- - const dataJsonlEntry = entries.find( - (entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"), - )!; + // Prefer root data.jsonl; fall back to nested for content validation even + // though we already reported the root-placement error above. + const dataJsonlEntry = hasRootDataJsonl + ? "data.jsonl" + : entries.find((entry) => entry.endsWith("/data.jsonl"))!; const tmpDir = join(tmpdir(), `bl-zip-${randomBytes(6).toString("hex")}`); mkdirSync(tmpDir, { recursive: true }); const tmpJsonl = join(tmpDir, "data.jsonl"); diff --git a/packages/core/src/finetune/profiles/cpt.ts b/packages/core/src/finetune/profiles/cpt.ts index ab5aba7..f962944 100644 --- a/packages/core/src/finetune/profiles/cpt.ts +++ b/packages/core/src/finetune/profiles/cpt.ts @@ -1,7 +1,39 @@ /** * `cpt` profile — Continual Pre-Training (full-parameter). * Maps to the server's `cpt` training type. CPT record schema. + * CPT allows larger files (300 MB) compared to SFT/DPO (200 MB). */ -import { textProfile } from "./common.ts"; +import type { TrainingProfile, DataModality } from "./types.ts"; +import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts"; +import { validateDataset } from "../../dataset/validate/registry.ts"; +import { resolveTextHyperParameters } from "./common.ts"; +import { MAX_CPT_BYTES } from "../../dataset/validate/common.ts"; -export const cptProfile = textProfile("cpt", "cpt", "cpt"); +export const cptProfile: TrainingProfile = { + clientTrainingType: "cpt", + serverTrainingType: "cpt", + acceptedExtensions: [".jsonl"], + + async validate( + filePath: string, + _modality: DataModality, + opts: ValidateOpts, + ): Promise { + return validateDataset(filePath, { ...opts, schema: "cpt", maxBytes: MAX_CPT_BYTES }); + }, + + resolveHyperParameters( + _modality: DataModality, + flags: Record, + ): Record { + return resolveTextHyperParameters(flags); + }, + + shouldSkipGate(_gate: string, _modality: DataModality): boolean { + return false; + }, + + shouldSkipCapabilityCheck(_modality: DataModality): boolean { + return false; + }, +}; diff --git a/packages/core/tests/dataset-validate.test.ts b/packages/core/tests/dataset-validate.test.ts index b4b2b52..c1e67c5 100644 --- a/packages/core/tests/dataset-validate.test.ts +++ b/packages/core/tests/dataset-validate.test.ts @@ -56,8 +56,6 @@ describe("validateDataset — DPO schema", () => { }); test('schema "dpo" requires both chosen and rejected on every record', async () => { - // A record with neither chosen nor rejected is SFT-shaped; under --schema dpo - // it must be flagged as missing both preferences. const p = file("sft_under_dpo.jsonl", [SFT_OK]); const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); expect(r.valid).toBe(false); @@ -107,6 +105,15 @@ describe("validateDataset — DPO schema", () => { const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); expect(r.valid).toBe(true); }); + + test("DPO messages ending with assistant → DPO_LAST_MSG_NOT_USER warning", async () => { + const p = file("dpo_last_asst.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"yo"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("DPO_LAST_MSG_NOT_USER"); + }); }); describe("validateDataset — CPT schema", () => { @@ -142,8 +149,6 @@ describe("validateDataset — CPT schema", () => { }); test("auto-detect routes a {text} record to CPT, not ChatML", async () => { - // A CPT record has no `messages`; under auto-detect it must NOT produce a - // ChatML MISSING_MESSAGES error — it should be validated as CPT and pass. const p = file("cpt_auto.jsonl", [CPT_OK]); const r = await validateDataset(p, { fullValidate: true }); expect(r.valid).toBe(true); @@ -151,8 +156,6 @@ describe("validateDataset — CPT schema", () => { }); test("SFT record with a stray text field still routes to ChatML", async () => { - // {messages, text} is ambiguous; CPT detect requires text AND no messages, - // so this falls through to ChatML and validates as SFT (text ignored). const p = file("mixed.jsonl", [ '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"yo"}],"text":"noise"}', ]); @@ -162,6 +165,271 @@ describe("validateDataset — CPT schema", () => { }); }); +describe("validateDataset — content array format", () => { + test("content as [{text}] array passes validation", async () => { + const p = file("content_arr.jsonl", [ + '{"messages":[{"role":"system","content":[{"text":"sys"}]},{"role":"user","content":[{"text":"hi"}]},{"role":"assistant","content":[{"text":"hello"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).toEqual([]); + }); + + test("content as plain string still passes (legacy format)", async () => { + const p = file("content_str.jsonl", [SFT_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with image item passes (VL multimodal)", async () => { + const p = file("content_img.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"image":"img1.jpg"}]},{"role":"assistant","content":[{"text":"a cat"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with video string item passes", async () => { + const p = file("content_vid.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"video":"vid1.mp4"}]},{"role":"assistant","content":[{"text":"a car"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with video frame list passes", async () => { + const p = file("content_frames.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"video":["0.jpg","1.jpg","2.jpg"]}]},{"role":"assistant","content":[{"text":"frames"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with invalid item (no text/image/video) → error", async () => { + const p = file("content_bad_item.jsonl", [ + '{"messages":[{"role":"user","content":[{"foo":"bar"}]},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("CONTENT_ITEM_NO_KNOWN_FIELD"); + }); + + test("content as number → INVALID_CONTENT error", async () => { + const p = file("content_num.jsonl", [ + '{"messages":[{"role":"user","content":42},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_CONTENT"); + }); + + test("empty content array → EMPTY_CONTENT_ARRAY error", async () => { + const p = file("content_empty_arr.jsonl", [ + '{"messages":[{"role":"user","content":[]},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("EMPTY_CONTENT_ARRAY"); + }); + + test("DPO with content array format passes", async () => { + const p = file("dpo_arr.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"hi"}]}],"chosen":{"role":"assistant","content":[{"text":"good"}]},"rejected":{"role":"assistant","content":[{"text":"bad"}]}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(true); + }); +}); + +describe("validateDataset — tool calling (function calling)", () => { + const TOOL_OK = JSON.stringify({ + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ], + messages: [ + { role: "user", content: [{ text: "weather in Beijing" }] }, + { + role: "assistant", + content: [{ text: "let me check" }], + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Beijing"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_1", content: [{ text: '{"weather":"sunny"}' }] }, + { role: "assistant", content: [{ text: "It is sunny." }] }, + ], + }); + + test("valid tool calling record passes", async () => { + const p = file("tool_ok.jsonl", [TOOL_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).toEqual([]); + }); + + test("tool role is accepted (no INVALID_ROLE)", async () => { + const p = file("tool_role.jsonl", [TOOL_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(codes(r).errors).not.toContain("INVALID_ROLE"); + }); + + test("tool message without tool_call_id → TOOL_MISSING_CALL_ID", async () => { + const p = file("tool_no_id.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","content":"result"},{"role":"assistant","content":"done"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("TOOL_MISSING_CALL_ID"); + }); + + test("tool_call_id unmatched → TOOL_CALL_ID_UNMATCHED warning", async () => { + const p = file("tool_unmatched.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","tool_call_id":"WRONG_ID","content":"result"},{"role":"assistant","content":"done"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("TOOL_CALL_ID_UNMATCHED"); + }); + + test("tool_calls with missing function name → TOOL_CALL_FN_NO_NAME", async () => { + const p = file("tool_no_fn_name.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"arguments":"{}"}}]},{"role":"tool","tool_call_id":"c1","content":"r"},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("TOOL_CALL_FN_NO_NAME"); + }); + + test("assistant with tool_calls but no content is valid", async () => { + const p = file("tool_no_content.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","tool_call_id":"c1","content":"r"},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).not.toContain("MISSING_CONTENT"); + }); +}); + +describe("validateDataset — thinking tags", () => { + test("think tag in last assistant is valid", async () => { + const p = file("think_ok.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"\\nreasoning\\n\\n\\nanswer"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).not.toContain("THINK_TAG_NOT_LAST"); + }); + + test("think tag in non-last assistant → THINK_TAG_NOT_LAST warning", async () => { + const p = file("think_mid.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"\\nearly\\n\\n\\nmid"},{"role":"user","content":"more"},{"role":"assistant","content":"final"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("THINK_TAG_NOT_LAST"); + }); + + test("think tag in content array format detected", async () => { + const p = file("think_arr.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"hi"}]},{"role":"assistant","content":[{"text":"\\nreason\\n\\n\\nans"}]},{"role":"user","content":[{"text":"more"}]},{"role":"assistant","content":[{"text":"final"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("THINK_TAG_NOT_LAST"); + }); +}); + +describe("validateDataset — OpenAI migration guards", () => { + test("message-level name field → UNSUPPORTED_FIELD_NAME warning", async () => { + const p = file("openai_name.jsonl", [ + '{"messages":[{"role":"user","content":"hi","name":"alice"},{"role":"assistant","content":"hello"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("UNSUPPORTED_FIELD_NAME"); + }); + + test("message-level weight field → UNSUPPORTED_FIELD_WEIGHT warning", async () => { + const p = file("openai_weight.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello","weight":0.5}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("UNSUPPORTED_FIELD_WEIGHT"); + }); + + test("record-level weight field → UNSUPPORTED_FIELD_WEIGHT warning", async () => { + const p = file("record_weight.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"weight":1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("UNSUPPORTED_FIELD_WEIGHT"); + }); +}); + +describe("validateDataset — loss_weight", () => { + test("valid loss_weight (0.5) passes", async () => { + const p = file("lw_ok.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":0.5}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).not.toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight out of range (1.5) → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_bad.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":1.5}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight negative → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_neg.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":-0.1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight non-number → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_str.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":"high"}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight boundary values 0 and 1 pass", async () => { + const p = file("lw_boundary.jsonl", [ + '{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"}],"loss_weight":0}', + '{"messages":[{"role":"user","content":"c"},{"role":"assistant","content":"d"}],"loss_weight":1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); +}); + describe("parseDatasetSchemaFlag", () => { test("undefined / empty → undefined (auto)", () => { expect(parseDatasetSchemaFlag(undefined)).toBeUndefined(); diff --git a/skills/bailian-finetune/reference/dataset.md b/skills/bailian-finetune/reference/dataset.md index 22cad6d..2a3b981 100644 --- a/skills/bailian-finetune/reference/dataset.md +++ b/skills/bailian-finetune/reference/dataset.md @@ -117,7 +117,7 @@ bl dataset list --output json | Flag | Type | Required | Description | | ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤200MB SFT/DPO, ≤300MB CPT, ≤2GB media zip) | | `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | | `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | | `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | @@ -128,13 +128,14 @@ bl dataset list --output json #### Notes - Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl -- manifest). Five record schemas are recognized: chatml = {messages:[...]} +- manifest). Six record schemas are recognized: chatml = {messages:[...]} - (SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."} - (continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav", - text:"..."} (audio fine-tuning); image = {img_path:"..."} (image -- generation). With no --schema, a record carrying wav_fn is validated as -- TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT, -- otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the +- generation); video = {first_frame_path:...} (video generation). With no +- --schema, a record carrying wav_fn is validated as TTS, img_path as image, +- chosen/rejected as DPO, text (no messages) as CPT, otherwise ChatML. +- Upload cap: 200MB SFT/DPO text, 300MB CPT, 2GB media zip. Upload uses the - OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is - persisted (the DashScope-native /api/v1/files drops it).