skills/modelstudioai__cli
1
0
Fork 0
mirror of https://github.com/modelstudioai/cli.git synced 2026-09-14 19:49:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): 支持多模态消息内容及图片URL数组

Browse Source 
- 扩展聊天消息内容类型，支持文本和图片URL的数组形式
- 处理 --image 参数，将图片URL作为多模态内容附加到最后一条用户消息
- 若无用户消息且指定图片URL，自动创建空用户消息以承载图片内容
- 禁止同时使用内嵌图片内容和 --image 参数，避免冲突
- 将知识搜索接口请求的图片参数字段 image_list 重命名为 images
- 单元测试覆盖多模态内容及图片数组行为验证
- 优化消息解析，支持JSON结构化消息和 role:content 格式
- 更新API类型声明，明确多模态消息结构与字段类型
This commit is contained in:
zeyu.fz
2026-06-29 13:36:32 +08:00
parent d2aa8cac17
commit 2ec2f34763
7 changed files with 170 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/tests/e2e/knowledge-chat.e2e.test.ts
+53 -6
View File
@@ -2,16 +2,21 @@ import { tmpdir } from "os";
import { describe, expect, test } from "vite-plus/test";
import { parseStdoutJson, runCli } from "./helpers.ts";
interface ContentPart {
type: string;
text?: string;
image_url?: { url: string };
}
interface DryRunBody {
endpoint?: string;
request?: {
input?: {
messages?: Array<{ role: string; content: string }>;
messages?: Array<{ role: string; content: string | ContentPart[] }>;
};
parameters?: {
agent_options?: {
agent_id?: string;
image_list?: string[];
};
};
stream?: boolean;
@@ -135,7 +140,7 @@ describe("e2e: knowledge chat", () => {
expect(msgs[2]?.content).toBe("它怎么工作");
});
test("--dry-run + --image 输出 image_list", async () => {
test("--dry-run + --image 输出多模态 content 数组", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
@@ -157,8 +162,50 @@ describe("e2e: knowledge chat", () => {
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
expect(data.request?.parameters?.agent_options?.image_list).toEqual([
"https://example.com/img.jpg",
]);
const lastMsg = data.request?.input?.messages?.[0];
expect(lastMsg?.role).toBe("user");
expect(Array.isArray(lastMsg?.content)).toBe(true);
const parts = lastMsg?.content as ContentPart[];
expect(parts[0]).toEqual({ type: "text", text: "描述这张图" });
expect(parts[1]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/img.jpg" },
});
});
test("--dry-run + --image 无 --message 自动创建空 user message", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
"chat",
"--dry-run",
"--agent-id",
"aid_test",
"--workspace-id",
"ws_test",
"--image",
"https://example.com/a.png",
"--image",
"https://example.com/b.png",
"--non-interactive",
"--output",
"json",
],
{ DASHSCOPE_API_KEY: "sk-fake-for-dryrun" },
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
const lastMsg = data.request?.input?.messages?.[0];
expect(lastMsg?.role).toBe("user");
const parts = lastMsg?.content as ContentPart[];
expect(parts[0]).toEqual({ type: "text", text: "" });
expect(parts[1]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/a.png" },
});
expect(parts[2]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/b.png" },
});
});
});
packages/cli/tests/e2e/knowledge-search.e2e.test.ts
+3 -3
View File
@@ -7,7 +7,7 @@ interface DryRunBody {
request?: {
query?: string;
agent_id?: string;
image_list?: string[];
images?: string[];
query_history?: Array<{ role: string; content: string }>;
};
}
@@ -96,7 +96,7 @@ describe("e2e: knowledge search", () => {
expect(data.request?.agent_id).toBe("aid_test");
});
test("--dry-run + --image 输出 image_list", async () => {
test("--dry-run + --image 输出 images", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
@@ -120,7 +120,7 @@ describe("e2e: knowledge search", () => {
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
expect(data.request?.image_list).toEqual([
expect(data.request?.images).toEqual([
"https://example.com/a.jpg",
"https://example.com/b.jpg",
]);
packages/commands/src/commands/knowledge/chat.ts
+95 -14
View File
@@ -9,22 +9,40 @@ import {
isInteractive,
type Config,
type GlobalFlags,
type KnowledgeChatContentPart,
type KnowledgeChatMessage,
type KnowledgeChatRequest,
type KnowledgeChatStreamChunk,
} from "bailian-cli-core";
import { failIfMissing, cmdUsage, emitResult, emitBare, promptText } from "bailian-cli-runtime";
interface ParsedMessage {
role: "user" | "assistant";
content: string;
}
function parseMessages(flags: GlobalFlags): ParsedMessage[] {
const messages: ParsedMessage[] = [];
/**
* Parse --message flags into KnowledgeChatMessage[].
* Supports:
* 1. Simple text: "hello" → {role:"user", content:"hello"}
* 2. Role prefix: "user:hello" / "assistant:hi" → {role, content}
* 3. JSON object: '{"role":"user","content":[...]}' → structured message (advanced)
*/
function parseMessages(flags: GlobalFlags): KnowledgeChatMessage[] {
const messages: KnowledgeChatMessage[] = [];
if (flags.message) {
const validRoles = new Set(["user", "assistant"]);
const msgs = flags.message as string[];
for (const m of msgs) {
// Try JSON object first (advanced usage)
if (m.startsWith("{")) {
try {
const parsed = JSON.parse(m) as { role?: string; content?: unknown };
if (parsed.role && validRoles.has(parsed.role) && parsed.content !== undefined) {
messages.push(parsed as KnowledgeChatMessage);
continue;
}
} catch {
// Not valid JSON, fall through to simple parsing
}
}
// Simple role:content or plain text
const colonIdx = m.indexOf(":");
const maybeRole = colonIdx !== -1 ? m.slice(0, colonIdx) : "";
@@ -38,6 +56,55 @@ function parseMessages(flags: GlobalFlags): ParsedMessage[] {
return messages;
}
/** Check if any message content already contains image_url parts */
function hasEmbeddedImages(messages: KnowledgeChatMessage[]): boolean {
for (const msg of messages) {
if (Array.isArray(msg.content)) {
if (msg.content.some((p) => p.type === "image_url")) return true;
}
}
return false;
}
/** Attach --image URLs to the last user message's content (as multimodal array) */
function attachImagesToLastUserMessage(
messages: KnowledgeChatMessage[],
imageUrls: string[],
): void {
// Find last user message index
let lastUserIdx = -1;
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i]!.role === "user") {
lastUserIdx = i;
break;
}
}
// If no user message exists, append an empty one
if (lastUserIdx === -1) {
messages.push({ role: "user", content: "" });
lastUserIdx = messages.length - 1;
}
const target = messages[lastUserIdx]!;
const contentParts: KnowledgeChatContentPart[] = [];
// Preserve existing text content (always include a text part, even if empty)
if (typeof target.content === "string") {
contentParts.push({ type: "text", text: target.content });
} else {
// Already an array, extend it
contentParts.push(...target.content);
}
// Append image parts
for (const url of imageUrls) {
contentParts.push({ type: "image_url", image_url: { url } });
}
target.content = contentParts;
}
/** SSE step_change → human-friendly progress label (TTY only) */
const STEP_LABELS: Record<string, string> = {
tool_calling: "🔍 Retrieving...",
@@ -67,7 +134,8 @@ export default defineCommand({
},
{
flag: "--image <url>",
description: "Image URL(s) (repeatable)",
description:
"Image URL (repeatable). Attached to the last user message as multimodal content",
type: "array",
},
],
@@ -80,12 +148,19 @@ export default defineCommand({
exampleArgs: [
'--message "What is RAG?" --agent-id aid-xxx --workspace-id ws-xxx',
'--message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx',
'--message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx',
],
async run(config: Config, flags: GlobalFlags) {
let messages = parseMessages(flags);
const imageUrls = flags.image as string[] | undefined;
const hasImages = imageUrls && imageUrls.length > 0;
if (messages.length === 0) {
if (isInteractive({ nonInteractive: config.nonInteractive })) {
if (hasImages) {
// --image without --message: create an empty user message to hold images
messages = [{ role: "user", content: "" }];
} else if (isInteractive({ nonInteractive: config.nonInteractive })) {
const hint = await promptText({ message: "Enter your message:" });
if (!hint) {
process.stderr.write("Chat cancelled.\n");
@@ -113,6 +188,17 @@ export default defineCommand({
// API only supports SSE; streamOutput controls whether to print tokens in real-time
const streamOutput = format === "text" && !!process.stdout.isTTY;
// Attach --image URLs to messages (multimodal content array)
if (hasImages) {
if (hasEmbeddedImages(messages)) {
throw new BailianError(
"Cannot use --image when messages already contain embedded image_url content parts. Use one approach or the other.",
ExitCode.USAGE,
);
}
attachImagesToLastUserMessage(messages, imageUrls!);
}
const body: KnowledgeChatRequest = {
input: {
messages,
@@ -125,11 +211,6 @@ export default defineCommand({
stream: true,
};
const imageUrls = flags.image as string[] | undefined;
if (imageUrls && imageUrls.length > 0) {
body.parameters.agent_options.image_list = imageUrls;
}
const url = knowledgeChatEndpoint(workspaceId);
if (config.dryRun) {
packages/commands/src/commands/knowledge/search.ts
+1 -1
View File
@@ -89,7 +89,7 @@ export default defineCommand({
const imageUrls = flags.image as string[] | undefined;
if (imageUrls && imageUrls.length > 0) {
body.image_list = imageUrls;
body.images = imageUrls;
}
// Parse query_history JSON for multi-turn context
packages/core/src/types/api.ts
+11 -4
View File
@@ -422,7 +422,7 @@ export interface DashScopeKnowledgeRetrieveResponse {
export interface KnowledgeSearchRequest {
query: string;
agent_id: string;
image_list?: string[];
images?: string[];
query_history?: Array<{ role: "user" | "assistant"; content: string }>;
}
@@ -456,15 +456,22 @@ export interface KnowledgeSearchResponse {
// ---- Knowledge Chat (新版 RAG 问答 SSE API, agent_id-based) ----
export type KnowledgeChatContentPart =
| { type: "text"; text: string }
| { type: "image_url"; image_url: { url: string } };
export interface KnowledgeChatMessage {
role: "user" | "assistant";
content: string | KnowledgeChatContentPart[];
}
export interface KnowledgeChatRequest {
input: {
messages: Array<{ role: "user" | "assistant"; content: string }>;
request_id?: string;
messages: KnowledgeChatMessage[];
};
parameters: {
agent_options: {
agent_id: string;
image_list?: string[];
user?: {
user_id?: string;
workspace_id?: string;
packages/core/src/types/index.ts
+2
View File
@@ -23,6 +23,8 @@ export type {
DashScopeVideoEditRequest,
DashScopeVideoRefRequest,
DashScopeVideoRequest,
KnowledgeChatContentPart,
KnowledgeChatMessage,
KnowledgeChatRequest,
KnowledgeChatStreamChunk,
KnowledgeRetrieveRequest,
skills/bailian-cli/reference/knowledge.md
+5 -1
View File
@@ -30,7 +30,7 @@ Index: [index.md](index.md)
| `--message <text>` | array | yes | Message text (repeatable). Supports role:content prefix to set role (e.g. user:hello), defaults to user. Follows OpenAI message format |
| `--agent-id <id>` | string | yes | Q&A service ID (find in console knowledge Q&A page) |
| `--workspace-id <id>` | string | no | Workspace ID for API endpoint URL (or set BAILIAN_WORKSPACE_ID) |
| `--image <url>` | array | no | Image URL(s) (repeatable) |
| `--image <url>` | array | no | Image URL (repeatable). Attached to the last user message as multimodal content |
#### Notes
@@ -49,6 +49,10 @@ bl knowledge chat --message "What is RAG?" --agent-id aid-xxx --workspace-id ws-
bl knowledge chat --message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx
```
```bash
bl knowledge chat --message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx
```
### `bl knowledge retrieve`
| Field | Value |