diff --git a/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts b/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts index a969b40..27813c4 100644 --- a/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts +++ b/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts @@ -2,16 +2,21 @@ import { tmpdir } from "os"; import { describe, expect, test } from "vite-plus/test"; import { parseStdoutJson, runCli } from "./helpers.ts"; +interface ContentPart { + type: string; + text?: string; + image_url?: { url: string }; +} + interface DryRunBody { endpoint?: string; request?: { input?: { - messages?: Array<{ role: string; content: string }>; + messages?: Array<{ role: string; content: string | ContentPart[] }>; }; parameters?: { agent_options?: { agent_id?: string; - image_list?: string[]; }; }; stream?: boolean; @@ -135,7 +140,7 @@ describe("e2e: knowledge chat", () => { expect(msgs[2]?.content).toBe("它怎么工作"); }); - test("--dry-run + --image 输出 image_list", async () => { + test("--dry-run + --image 输出多模态 content 数组", async () => { const { stdout, stderr, exitCode } = await runCli( [ "knowledge", @@ -157,8 +162,50 @@ describe("e2e: knowledge chat", () => { ); expect(exitCode, stderr).toBe(0); const data = parseStdoutJson(stdout); - expect(data.request?.parameters?.agent_options?.image_list).toEqual([ - "https://example.com/img.jpg", - ]); + const lastMsg = data.request?.input?.messages?.[0]; + expect(lastMsg?.role).toBe("user"); + expect(Array.isArray(lastMsg?.content)).toBe(true); + const parts = lastMsg?.content as ContentPart[]; + expect(parts[0]).toEqual({ type: "text", text: "描述这张图" }); + expect(parts[1]).toEqual({ + type: "image_url", + image_url: { url: "https://example.com/img.jpg" }, + }); + }); + + test("--dry-run + --image 无 --message 自动创建空 user message", async () => { + const { stdout, stderr, exitCode } = await runCli( + [ + "knowledge", + "chat", + "--dry-run", + "--agent-id", + "aid_test", + "--workspace-id", + "ws_test", + "--image", + "https://example.com/a.png", + "--image", + "https://example.com/b.png", + "--non-interactive", + "--output", + "json", + ], + { DASHSCOPE_API_KEY: "sk-fake-for-dryrun" }, + ); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson(stdout); + const lastMsg = data.request?.input?.messages?.[0]; + expect(lastMsg?.role).toBe("user"); + const parts = lastMsg?.content as ContentPart[]; + expect(parts[0]).toEqual({ type: "text", text: "" }); + expect(parts[1]).toEqual({ + type: "image_url", + image_url: { url: "https://example.com/a.png" }, + }); + expect(parts[2]).toEqual({ + type: "image_url", + image_url: { url: "https://example.com/b.png" }, + }); }); }); diff --git a/packages/cli/tests/e2e/knowledge-search.e2e.test.ts b/packages/cli/tests/e2e/knowledge-search.e2e.test.ts index 785f0a9..e611af5 100644 --- a/packages/cli/tests/e2e/knowledge-search.e2e.test.ts +++ b/packages/cli/tests/e2e/knowledge-search.e2e.test.ts @@ -7,7 +7,7 @@ interface DryRunBody { request?: { query?: string; agent_id?: string; - image_list?: string[]; + images?: string[]; query_history?: Array<{ role: string; content: string }>; }; } @@ -96,7 +96,7 @@ describe("e2e: knowledge search", () => { expect(data.request?.agent_id).toBe("aid_test"); }); - test("--dry-run + --image 输出 image_list", async () => { + test("--dry-run + --image 输出 images", async () => { const { stdout, stderr, exitCode } = await runCli( [ "knowledge", @@ -120,7 +120,7 @@ describe("e2e: knowledge search", () => { ); expect(exitCode, stderr).toBe(0); const data = parseStdoutJson(stdout); - expect(data.request?.image_list).toEqual([ + expect(data.request?.images).toEqual([ "https://example.com/a.jpg", "https://example.com/b.jpg", ]); diff --git a/packages/commands/src/commands/knowledge/chat.ts b/packages/commands/src/commands/knowledge/chat.ts index 59950c9..f52f64a 100644 --- a/packages/commands/src/commands/knowledge/chat.ts +++ b/packages/commands/src/commands/knowledge/chat.ts @@ -9,22 +9,40 @@ import { isInteractive, type Config, type GlobalFlags, + type KnowledgeChatContentPart, + type KnowledgeChatMessage, type KnowledgeChatRequest, type KnowledgeChatStreamChunk, } from "bailian-cli-core"; import { failIfMissing, cmdUsage, emitResult, emitBare, promptText } from "bailian-cli-runtime"; -interface ParsedMessage { - role: "user" | "assistant"; - content: string; -} - -function parseMessages(flags: GlobalFlags): ParsedMessage[] { - const messages: ParsedMessage[] = []; +/** + * Parse --message flags into KnowledgeChatMessage[]. + * Supports: + * 1. Simple text: "hello" → {role:"user", content:"hello"} + * 2. Role prefix: "user:hello" / "assistant:hi" → {role, content} + * 3. JSON object: '{"role":"user","content":[...]}' → structured message (advanced) + */ +function parseMessages(flags: GlobalFlags): KnowledgeChatMessage[] { + const messages: KnowledgeChatMessage[] = []; if (flags.message) { const validRoles = new Set(["user", "assistant"]); const msgs = flags.message as string[]; for (const m of msgs) { + // Try JSON object first (advanced usage) + if (m.startsWith("{")) { + try { + const parsed = JSON.parse(m) as { role?: string; content?: unknown }; + if (parsed.role && validRoles.has(parsed.role) && parsed.content !== undefined) { + messages.push(parsed as KnowledgeChatMessage); + continue; + } + } catch { + // Not valid JSON, fall through to simple parsing + } + } + + // Simple role:content or plain text const colonIdx = m.indexOf(":"); const maybeRole = colonIdx !== -1 ? m.slice(0, colonIdx) : ""; @@ -38,6 +56,55 @@ function parseMessages(flags: GlobalFlags): ParsedMessage[] { return messages; } +/** Check if any message content already contains image_url parts */ +function hasEmbeddedImages(messages: KnowledgeChatMessage[]): boolean { + for (const msg of messages) { + if (Array.isArray(msg.content)) { + if (msg.content.some((p) => p.type === "image_url")) return true; + } + } + return false; +} + +/** Attach --image URLs to the last user message's content (as multimodal array) */ +function attachImagesToLastUserMessage( + messages: KnowledgeChatMessage[], + imageUrls: string[], +): void { + // Find last user message index + let lastUserIdx = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i]!.role === "user") { + lastUserIdx = i; + break; + } + } + + // If no user message exists, append an empty one + if (lastUserIdx === -1) { + messages.push({ role: "user", content: "" }); + lastUserIdx = messages.length - 1; + } + + const target = messages[lastUserIdx]!; + const contentParts: KnowledgeChatContentPart[] = []; + + // Preserve existing text content (always include a text part, even if empty) + if (typeof target.content === "string") { + contentParts.push({ type: "text", text: target.content }); + } else { + // Already an array, extend it + contentParts.push(...target.content); + } + + // Append image parts + for (const url of imageUrls) { + contentParts.push({ type: "image_url", image_url: { url } }); + } + + target.content = contentParts; +} + /** SSE step_change → human-friendly progress label (TTY only) */ const STEP_LABELS: Record = { tool_calling: "🔍 Retrieving...", @@ -67,7 +134,8 @@ export default defineCommand({ }, { flag: "--image ", - description: "Image URL(s) (repeatable)", + description: + "Image URL (repeatable). Attached to the last user message as multimodal content", type: "array", }, ], @@ -80,12 +148,19 @@ export default defineCommand({ exampleArgs: [ '--message "What is RAG?" --agent-id aid-xxx --workspace-id ws-xxx', '--message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx', + '--message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx', ], async run(config: Config, flags: GlobalFlags) { let messages = parseMessages(flags); + const imageUrls = flags.image as string[] | undefined; + const hasImages = imageUrls && imageUrls.length > 0; + if (messages.length === 0) { - if (isInteractive({ nonInteractive: config.nonInteractive })) { + if (hasImages) { + // --image without --message: create an empty user message to hold images + messages = [{ role: "user", content: "" }]; + } else if (isInteractive({ nonInteractive: config.nonInteractive })) { const hint = await promptText({ message: "Enter your message:" }); if (!hint) { process.stderr.write("Chat cancelled.\n"); @@ -113,6 +188,17 @@ export default defineCommand({ // API only supports SSE; streamOutput controls whether to print tokens in real-time const streamOutput = format === "text" && !!process.stdout.isTTY; + // Attach --image URLs to messages (multimodal content array) + if (hasImages) { + if (hasEmbeddedImages(messages)) { + throw new BailianError( + "Cannot use --image when messages already contain embedded image_url content parts. Use one approach or the other.", + ExitCode.USAGE, + ); + } + attachImagesToLastUserMessage(messages, imageUrls!); + } + const body: KnowledgeChatRequest = { input: { messages, @@ -125,11 +211,6 @@ export default defineCommand({ stream: true, }; - const imageUrls = flags.image as string[] | undefined; - if (imageUrls && imageUrls.length > 0) { - body.parameters.agent_options.image_list = imageUrls; - } - const url = knowledgeChatEndpoint(workspaceId); if (config.dryRun) { diff --git a/packages/commands/src/commands/knowledge/search.ts b/packages/commands/src/commands/knowledge/search.ts index ae3fadf..e869455 100644 --- a/packages/commands/src/commands/knowledge/search.ts +++ b/packages/commands/src/commands/knowledge/search.ts @@ -89,7 +89,7 @@ export default defineCommand({ const imageUrls = flags.image as string[] | undefined; if (imageUrls && imageUrls.length > 0) { - body.image_list = imageUrls; + body.images = imageUrls; } // Parse query_history JSON for multi-turn context diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts index 6a5ad77..d7a570c 100644 --- a/packages/core/src/types/api.ts +++ b/packages/core/src/types/api.ts @@ -422,7 +422,7 @@ export interface DashScopeKnowledgeRetrieveResponse { export interface KnowledgeSearchRequest { query: string; agent_id: string; - image_list?: string[]; + images?: string[]; query_history?: Array<{ role: "user" | "assistant"; content: string }>; } @@ -456,15 +456,22 @@ export interface KnowledgeSearchResponse { // ---- Knowledge Chat (新版 RAG 问答 SSE API, agent_id-based) ---- +export type KnowledgeChatContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } }; + +export interface KnowledgeChatMessage { + role: "user" | "assistant"; + content: string | KnowledgeChatContentPart[]; +} + export interface KnowledgeChatRequest { input: { - messages: Array<{ role: "user" | "assistant"; content: string }>; - request_id?: string; + messages: KnowledgeChatMessage[]; }; parameters: { agent_options: { agent_id: string; - image_list?: string[]; user?: { user_id?: string; workspace_id?: string; diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index 6495a93..fd01b48 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -23,6 +23,8 @@ export type { DashScopeVideoEditRequest, DashScopeVideoRefRequest, DashScopeVideoRequest, + KnowledgeChatContentPart, + KnowledgeChatMessage, KnowledgeChatRequest, KnowledgeChatStreamChunk, KnowledgeRetrieveRequest, diff --git a/skills/bailian-cli/reference/knowledge.md b/skills/bailian-cli/reference/knowledge.md index 09e85d9..94c2935 100644 --- a/skills/bailian-cli/reference/knowledge.md +++ b/skills/bailian-cli/reference/knowledge.md @@ -30,7 +30,7 @@ Index: [index.md](index.md) | `--message ` | array | yes | Message text (repeatable). Supports role:content prefix to set role (e.g. user:hello), defaults to user. Follows OpenAI message format | | `--agent-id ` | string | yes | Q&A service ID (find in console knowledge Q&A page) | | `--workspace-id ` | string | no | Workspace ID for API endpoint URL (or set BAILIAN_WORKSPACE_ID) | -| `--image ` | array | no | Image URL(s) (repeatable) | +| `--image ` | array | no | Image URL (repeatable). Attached to the last user message as multimodal content | #### Notes @@ -49,6 +49,10 @@ bl knowledge chat --message "What is RAG?" --agent-id aid-xxx --workspace-id ws- bl knowledge chat --message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx ``` +```bash +bl knowledge chat --message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx +``` + ### `bl knowledge retrieve` | Field | Value |