diff --git a/packages/commands/tests/e2e/knowledge/journeys/README.md b/packages/commands/tests/e2e/knowledge/journeys/README.md index 2942143..210074a 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/README.md +++ b/packages/commands/tests/e2e/knowledge/journeys/README.md @@ -10,7 +10,7 @@ | J1 | 冷启动：建库并获得首个答案 | `j1-cold-start.e2e.test.ts` | retrieve/search(beta) 召回标记词（硬）；chat 回答引用标记词（软） | | J2 | 内容运维：文档增删的可见性 | `j2-content-ops.e2e.test.ts` | 双标记词命中 → 删除其一后 markerB 消失且 markerA 仍在（硬） | | J3 | 检索精修：chunk 排除生效 | `j3-chunk-tuning.e2e.test.ts` | exclude 后排除标志生效（硬）；include 恢复（软）。retrieve 不过滤被排除 chunk，以 `is_displayed_chunk_content` 标志为准 | -| J4 | 服务调优：草稿→修改→发布 | `j4-service-tuning.e2e.test.ts` | beta 草稿可用、update 落库、发布后正式版可用（硬） | +| J4 | 服务调优：草稿→修改→发布 | `j4-service-tuning.e2e.test.ts` | beta 草稿可用、update 落库、发布后正式版详情携带修改且可用（硬） | | J5 | 数据面治理：collection/category/file | `j5-data-plane.e2e.test.ts` | 自建类目内文件可见/可删，类目删后消失（硬） | | J6 | 退场清理：删库验证消失 | 复用 [`../knowledge-kb-delete.e2e.test.ts`](../knowledge-kb-delete.e2e.test.ts) live 链 | delete 后 list 不再包含（硬） | diff --git a/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts index c2ac66b..007932b 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts +++ b/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts @@ -10,6 +10,7 @@ import { cleanupKbFixture, createJourneyReporter, createKbWithDocs, + nodesRecallMarker, patchSearchServiceRetrievalConfig, pollUntil, uniqueMarker, @@ -44,7 +45,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J1: 冷启动首答 (live, 自清 "--output", "json", ]), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 180_000, intervalMs: 15_000 }, ); reporter.recordNote(`retrieve 轮询 ${retrievePoll.attempts} 次`); @@ -100,7 +101,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J1: 冷启动首答 (live, 自清 "--output", "json", ]), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 120_000, intervalMs: 15_000 }, ); reporter.recordNote(`search 轮询 ${searchPoll.attempts} 次`); diff --git a/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts index 3a67cb0..89a586c 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts +++ b/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts @@ -10,6 +10,7 @@ import { cleanupKbFixture, createJourneyReporter, createKbWithDocs, + nodesRecallMarker, pollUntil, uniqueMarker, type KbFixture, @@ -113,7 +114,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J2: 内容运维 (live, 自清理 "--output", "json", ]), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 180_000, intervalMs: 15_000 }, ); const pollA = await retrieveMarker(markerA, "retrieve markerA"); @@ -165,7 +166,10 @@ describe.skipIf(!isKbAdminE2EReady())("journey J2: 内容运维 (live, 自清理 "json", ]); expect(stillA.exitCode, stillA.stderr).toBe(0); - expect(stillA.stdout, `doc delete 误伤: ${markerA} 不再召回`).toContain(markerA); + expect( + nodesRecallMarker(stillA.stdout, markerA), + `doc delete 误伤: ${markerA} 不再召回`, + ).toBe(true); } finally { await cleanupKbFixture(reporter, JOURNEY_J2_ROUTES, fixture, workspaceId); reporter.finalize(); diff --git a/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts index ba02233..fe1a1e4 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts +++ b/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts @@ -12,6 +12,7 @@ import { cleanupKbFixture, createJourneyReporter, createKbWithDocs, + nodesRecallMarker, pollUntil, uniqueMarker, type KbFixture, @@ -75,7 +76,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J3: 检索精修 (live, 自清理 ]; const baseline = await pollUntil( () => reporter.runStep("retrieve baseline", JOURNEY_J3_ROUTES, retrieveArgs), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 180_000, intervalMs: 15_000 }, ); expect(baseline.satisfied, `基线未召回标记词 ${marker}`).toBe(true); diff --git a/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts index abf0cc8..671648b 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts +++ b/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts @@ -1,6 +1,12 @@ -// J4 service tuning: draft (beta) usable → config change persisted → released version usable after deploy. +// J4 service tuning: draft (beta) usable → config change persisted → released version carries the change after deploy. // create + service create (search, initial draft/beta) → search --agent-version beta recalls (hard) -// → service update --description → service get asserts the change (hard) → deploy → released search recalls (hard). +// → service update --description/--temperature (scalar merge path) → get (beta) asserts (hard) +// → service update --config-file tweaking kb_search_configs (whole-replace path) → get (beta) asserts +// the nested change AND that the replace kept the scalar tuning (hard) → deploy +// → get (released version) asserts both changes landed in the published version (hard) → released search recalls (hard). +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, test } from "vite-plus/test"; import { isKbAdminE2EReady, parseStdoutJson } from "../../helpers.ts"; import { JOURNEY_J4_ROUTES } from "../../topic-routes.ts"; @@ -8,6 +14,7 @@ import { cleanupKbFixture, createJourneyReporter, createKbWithDocs, + nodesRecallMarker, patchSearchServiceRetrievalConfig, pollUntil, uniqueMarker, @@ -17,7 +24,7 @@ import { describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自清理)", () => { const workspaceId = process.env.BAILIAN_WORKSPACE_ID!; - test("draft 可用 → update 落库 → deploy → 正式版可用", async () => { + test("draft 可用 → update 落库 → deploy → 正式版详情携带修改且可用", async () => { const reporter = createJourneyReporter(import.meta.url); const marker = uniqueMarker("j4"); const fixture: Partial = {}; @@ -71,25 +78,33 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自 ]; const betaPoll = await pollUntil( () => reporter.runStep("search (beta)", JOURNEY_J4_ROUTES, betaSearchArgs), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 180_000, intervalMs: 15_000 }, ); reporter.recordNote(`beta search 轮询 ${betaPoll.attempts} 次`); expect(betaPoll.satisfied, `search(beta) 未召回标记词 ${marker}`).toBe(true); - // 3) update the description (top-level scalar, valid for the search scene too) → get asserts persistence (hard) + // 3) tune the draft: description (top-level) + temperature (config-level scalar, + // read-merge-write keeps the backfilled retrieval params) → get asserts both (hard) const newDescription = `journey j4 tuned at ${Date.now()}`; - const updateRun = await reporter.runStep("service update --description", JOURNEY_J4_ROUTES, [ - "knowledge", - "service", - "update", - "--agent-id", - agentId, - "--description", - newDescription, - "--workspace-id", - workspaceId, - ]); + const tunedTemperature = 0.55; + const updateRun = await reporter.runStep( + "service update --description --temperature", + JOURNEY_J4_ROUTES, + [ + "knowledge", + "service", + "update", + "--agent-id", + agentId, + "--description", + newDescription, + "--temperature", + String(tunedTemperature), + "--workspace-id", + workspaceId, + ], + ); expect(updateRun.exitCode, updateRun.stderr).toBe(0); const getRun = await reporter.runStep("service get (beta)", JOURNEY_J4_ROUTES, [ @@ -106,10 +121,88 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自 "json", ]); expect(getRun.exitCode, getRun.stderr).toBe(0); - const getData = parseStdoutJson<{ data?: { agent_desc?: string } }>(getRun.stdout); + const getData = parseStdoutJson<{ + data?: { + agent_desc?: string; + agent_details?: Array<{ + agent_config?: { temperature?: number } & Record; + }>; + }; + }>(getRun.stdout); expect(getData.data?.agent_desc).toBe(newDescription); + expect(getData.data?.agent_details?.[0]?.agent_config?.temperature).toBe(tunedTemperature); - // 4) deploy → released search (without --agent-version) recalls (hard) + // 3.5) complex nested tuning via --config-file (whole-replace path, distinct + // from the scalar merge path above): read the current beta config, tweak + // kb_search_configs, write it back — then assert the nested change landed + // AND the replace kept the scalar tuning intact + const tunedDenseTopK = 66; // distinctive value, still recall-safe (loose top-k) + const betaConfig = getData.data?.agent_details?.[0]?.agent_config as + | ({ kb_search_configs?: Array> } & Record) + | undefined; + expect(betaConfig?.kb_search_configs?.length, "beta 配置应含 kb_search_configs").toBeTruthy(); + for (const kbConfig of betaConfig!.kb_search_configs!) { + kbConfig.dense_similarity_top_k = tunedDenseTopK; + } + const configDir = mkdtempSync(join(tmpdir(), "j4-config-")); + const configFile = join(configDir, "agent-config.json"); + writeFileSync(configFile, JSON.stringify(betaConfig)); + const configUpdateRun = await reporter.runStep( + "service update --config-file (kb_search_configs)", + JOURNEY_J4_ROUTES, + [ + "knowledge", + "service", + "update", + "--agent-id", + agentId, + "--config-file", + configFile, + "--workspace-id", + workspaceId, + ], + ); + expect(configUpdateRun.exitCode, configUpdateRun.stderr).toBe(0); + + const betaAfterConfigRun = await reporter.runStep( + "service get (beta, after config-file)", + JOURNEY_J4_ROUTES, + [ + "knowledge", + "service", + "get", + "--agent-id", + agentId, + "--agent-version", + "beta", + "--workspace-id", + workspaceId, + "--output", + "json", + ], + ); + expect(betaAfterConfigRun.exitCode, betaAfterConfigRun.stderr).toBe(0); + const betaAfterConfig = parseStdoutJson<{ + data?: { + agent_details?: Array<{ + agent_config?: { + temperature?: number; + kb_search_configs?: Array<{ dense_similarity_top_k?: number }>; + }; + }>; + }; + }>(betaAfterConfigRun.stdout).data?.agent_details?.[0]?.agent_config; + expect( + betaAfterConfig?.kb_search_configs?.[0]?.dense_similarity_top_k, + "kb_search_configs 嵌套修改未落库", + ).toBe(tunedDenseTopK); + expect(betaAfterConfig?.temperature, "config-file 整体替换不应冲掉已调优的 temperature").toBe( + tunedTemperature, + ); + + // 4) deploy → the published version's detail must carry the tuned config (hard): + // search alone only proves the released service responds, not that the + // change actually shipped const deployRun = await reporter.runStep("service deploy", JOURNEY_J4_ROUTES, [ "knowledge", "service", @@ -122,8 +215,52 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自 "--quiet", ]); expect(deployRun.exitCode, deployRun.stderr).toBe(0); - reporter.recordNote(`deploy 版本号: ${deployRun.stdout.trim().split("\n").pop() ?? "?"}`); + const deployedVersion = deployRun.stdout.trim().split("\n").pop() ?? ""; + expect(deployedVersion, "deploy 应输出新版本号").toBeTruthy(); + reporter.recordNote(`deploy 版本号: ${deployedVersion}`); + const releasedGetRun = await reporter.runStep( + `service get (released v${deployedVersion})`, + JOURNEY_J4_ROUTES, + [ + "knowledge", + "service", + "get", + "--agent-id", + agentId, + "--agent-version", + deployedVersion, + "--workspace-id", + workspaceId, + "--output", + "json", + ], + ); + expect(releasedGetRun.exitCode, releasedGetRun.stderr).toBe(0); + const releasedData = parseStdoutJson<{ + data?: { + agent_details?: Array<{ + agent_version?: string; + agent_config?: { + temperature?: number; + kb_search_configs?: Array<{ dense_similarity_top_k?: number }>; + }; + }>; + }; + }>(releasedGetRun.stdout); + const releasedDetail = releasedData.data?.agent_details?.find( + (detail) => detail.agent_version === deployedVersion, + ); + expect(releasedDetail, `get 应返回已发布版本 ${deployedVersion} 的详情`).toBeTruthy(); + expect(releasedDetail?.agent_config?.temperature, "调优的 temperature 未进入正式版配置").toBe( + tunedTemperature, + ); + expect( + releasedDetail?.agent_config?.kb_search_configs?.[0]?.dense_similarity_top_k, + "调优的 kb_search_configs 未进入正式版配置", + ).toBe(tunedDenseTopK); + + // 5) released search (without --agent-version) recalls (hard) const releasedPoll = await pollUntil( () => reporter.runStep("search (released)", JOURNEY_J4_ROUTES, [ @@ -138,7 +275,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自 "--output", "json", ]), - (run) => run.exitCode === 0 && run.stdout.includes(marker), + (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker), { timeoutMs: 120_000, intervalMs: 15_000 }, ); reporter.recordNote(`正式版 search 轮询 ${releasedPoll.attempts} 次`); diff --git a/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts b/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts index 3e1c058..192b410 100644 --- a/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts +++ b/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts @@ -377,6 +377,29 @@ export async function cleanupKbFixture( } } +// ---- Marker-recall predicate (strict) ---- + +/** + * Positive recall check for retrieve/search JSON output: the marker must appear + * inside a returned node's text (not merely anywhere in the raw output — doc_name + * or other metadata echoing the marker must not count as recall). + * Negative assertions ("marker gone") should keep scanning the whole stdout: + * absence from the entire output is the stronger guarantee. + */ +export function nodesRecallMarker(stdout: string, marker: string): boolean { + try { + const payload = JSON.parse(stdout) as { + data?: { nodes?: Array<{ text?: string; metadata?: { content?: string } }> }; + }; + return (payload.data?.nodes ?? []).some((node) => + (node.metadata?.content ?? node.text ?? "").includes(marker), + ); + } catch { + // Mid-poll runs may fail before emitting JSON — treat as "not recalled yet" + return false; + } +} + // ---- Backfill required retrieval parameters for search services (server gotcha) ---- // The minimal kb_search_configs created by service create --index-id is missing // required fields like rerank_min_score / dense_similarity_top_k, so the search