feat(cli): 支持多模态消息内容及图片URL数组

- 扩展聊天消息内容类型,支持文本和图片URL的数组形式
- 处理 --image 参数,将图片URL作为多模态内容附加到最后一条用户消息
- 若无用户消息且指定图片URL,自动创建空用户消息以承载图片内容
- 禁止同时使用内嵌图片内容和 --image 参数,避免冲突
- 将知识搜索接口请求的图片参数字段 image_list 重命名为 images
- 单元测试覆盖多模态内容及图片数组行为验证
- 优化消息解析,支持JSON结构化消息和 role:content 格式
- 更新API类型声明,明确多模态消息结构与字段类型
This commit is contained in:
zeyu.fz
2026-06-29 13:36:32 +08:00
parent d2aa8cac17
commit 2ec2f34763
7 changed files with 170 additions and 29 deletions
@@ -2,16 +2,21 @@ import { tmpdir } from "os";
import { describe, expect, test } from "vite-plus/test";
import { parseStdoutJson, runCli } from "./helpers.ts";
interface ContentPart {
type: string;
text?: string;
image_url?: { url: string };
}
interface DryRunBody {
endpoint?: string;
request?: {
input?: {
messages?: Array<{ role: string; content: string }>;
messages?: Array<{ role: string; content: string | ContentPart[] }>;
};
parameters?: {
agent_options?: {
agent_id?: string;
image_list?: string[];
};
};
stream?: boolean;
@@ -135,7 +140,7 @@ describe("e2e: knowledge chat", () => {
expect(msgs[2]?.content).toBe("它怎么工作");
});
test("--dry-run + --image 输出 image_list", async () => {
test("--dry-run + --image 输出多模态 content 数组", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
@@ -157,8 +162,50 @@ describe("e2e: knowledge chat", () => {
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
expect(data.request?.parameters?.agent_options?.image_list).toEqual([
"https://example.com/img.jpg",
]);
const lastMsg = data.request?.input?.messages?.[0];
expect(lastMsg?.role).toBe("user");
expect(Array.isArray(lastMsg?.content)).toBe(true);
const parts = lastMsg?.content as ContentPart[];
expect(parts[0]).toEqual({ type: "text", text: "描述这张图" });
expect(parts[1]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/img.jpg" },
});
});
test("--dry-run + --image 无 --message 自动创建空 user message", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
"chat",
"--dry-run",
"--agent-id",
"aid_test",
"--workspace-id",
"ws_test",
"--image",
"https://example.com/a.png",
"--image",
"https://example.com/b.png",
"--non-interactive",
"--output",
"json",
],
{ DASHSCOPE_API_KEY: "sk-fake-for-dryrun" },
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
const lastMsg = data.request?.input?.messages?.[0];
expect(lastMsg?.role).toBe("user");
const parts = lastMsg?.content as ContentPart[];
expect(parts[0]).toEqual({ type: "text", text: "" });
expect(parts[1]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/a.png" },
});
expect(parts[2]).toEqual({
type: "image_url",
image_url: { url: "https://example.com/b.png" },
});
});
});
@@ -7,7 +7,7 @@ interface DryRunBody {
request?: {
query?: string;
agent_id?: string;
image_list?: string[];
images?: string[];
query_history?: Array<{ role: string; content: string }>;
};
}
@@ -96,7 +96,7 @@ describe("e2e: knowledge search", () => {
expect(data.request?.agent_id).toBe("aid_test");
});
test("--dry-run + --image 输出 image_list", async () => {
test("--dry-run + --image 输出 images", async () => {
const { stdout, stderr, exitCode } = await runCli(
[
"knowledge",
@@ -120,7 +120,7 @@ describe("e2e: knowledge search", () => {
);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<DryRunBody>(stdout);
expect(data.request?.image_list).toEqual([
expect(data.request?.images).toEqual([
"https://example.com/a.jpg",
"https://example.com/b.jpg",
]);