test(e2e): 优化检索结果标记召回判断并完善服务调优用例

clark-fc · clark-fc · commit ef463e8d5ddc · 2026-08-05T19:56:52.000+08:00
- 新增节点召回标记判断函数 nodesRecallMarker，避免误判 marker 出现位置
- 将所有相关轮询断言替换为基于 nodesRecallMarker 的更严格判断
- 调整删除测试中对误伤判断的断言逻辑，确保准确检测召回标记
- 扩展服务调优用例，增加描述和温度参数调优测试，验证配置持久化
- 添加通过配置文件更新 kb_search_configs 并校验嵌套配置修改生效
- 部署后验证发布版本配置正确包含所有调优项
- 更新流程注释与断言提示，提升测试用例可读性和覆盖度
diff --git a/packages/commands/tests/e2e/knowledge/journeys/README.md b/packages/commands/tests/e2e/knowledge/journeys/README.md
@@ -10,7 +10,7 @@
 | J1  | 冷启动：建库并获得首个答案           | `j1-cold-start.e2e.test.ts`                                                             | retrieve/search(beta) 召回标记词（硬）；chat 回答引用标记词（软）                                                       |
 | J2  | 内容运维：文档增删的可见性           | `j2-content-ops.e2e.test.ts`                                                            | 双标记词命中 → 删除其一后 markerB 消失且 markerA 仍在（硬）                                                             |
 | J3  | 检索精修：chunk 排除生效             | `j3-chunk-tuning.e2e.test.ts`                                                           | exclude 后排除标志生效（硬）；include 恢复（软）。retrieve 不过滤被排除 chunk，以 `is_displayed_chunk_content` 标志为准 |
-| J4  | 服务调优：草稿→修改→发布             | `j4-service-tuning.e2e.test.ts`                                                         | beta 草稿可用、update 落库、发布后正式版可用（硬）                                                                      |
+| J4  | 服务调优：草稿→修改→发布             | `j4-service-tuning.e2e.test.ts`                                                         | beta 草稿可用、update 落库、发布后正式版详情携带修改且可用（硬）                                                        |
 | J5  | 数据面治理：collection/category/file | `j5-data-plane.e2e.test.ts`                                                             | 自建类目内文件可见/可删，类目删后消失（硬）                                                                             |
 | J6  | 退场清理：删库验证消失               | 复用 [`../knowledge-kb-delete.e2e.test.ts`](../knowledge-kb-delete.e2e.test.ts) live 链 | delete 后 list 不再包含（硬）                                                                                           |
 
diff --git a/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j1-cold-start.e2e.test.ts
@@ -10,6 +10,7 @@ import {
   cleanupKbFixture,
   createJourneyReporter,
   createKbWithDocs,
+  nodesRecallMarker,
   patchSearchServiceRetrievalConfig,
   pollUntil,
   uniqueMarker,
@@ -44,7 +45,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J1: 冷启动首答 (live, 自清
             "--output",
             "json",
           ]),
-        (run) => run.exitCode === 0 && run.stdout.includes(marker),
+        (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
         { timeoutMs: 180_000, intervalMs: 15_000 },
       );
       reporter.recordNote(`retrieve 轮询 ${retrievePoll.attempts} 次`);
@@ -100,7 +101,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J1: 冷启动首答 (live, 自清
             "--output",
             "json",
           ]),
-        (run) => run.exitCode === 0 && run.stdout.includes(marker),
+        (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
         { timeoutMs: 120_000, intervalMs: 15_000 },
       );
       reporter.recordNote(`search 轮询 ${searchPoll.attempts} 次`);
diff --git a/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j2-content-ops.e2e.test.ts
@@ -10,6 +10,7 @@ import {
   cleanupKbFixture,
   createJourneyReporter,
   createKbWithDocs,
+  nodesRecallMarker,
   pollUntil,
   uniqueMarker,
   type KbFixture,
@@ -113,7 +114,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J2: 内容运维 (live, 自清理
               "--output",
               "json",
             ]),
-          (run) => run.exitCode === 0 && run.stdout.includes(marker),
+          (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
           { timeoutMs: 180_000, intervalMs: 15_000 },
         );
       const pollA = await retrieveMarker(markerA, "retrieve markerA");
@@ -165,7 +166,10 @@ describe.skipIf(!isKbAdminE2EReady())("journey J2: 内容运维 (live, 自清理
         "json",
       ]);
       expect(stillA.exitCode, stillA.stderr).toBe(0);
-      expect(stillA.stdout, `doc delete 误伤: ${markerA} 不再召回`).toContain(markerA);
+      expect(
+        nodesRecallMarker(stillA.stdout, markerA),
+        `doc delete 误伤: ${markerA} 不再召回`,
+      ).toBe(true);
     } finally {
       await cleanupKbFixture(reporter, JOURNEY_J2_ROUTES, fixture, workspaceId);
       reporter.finalize();
diff --git a/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j3-chunk-tuning.e2e.test.ts
@@ -12,6 +12,7 @@ import {
   cleanupKbFixture,
   createJourneyReporter,
   createKbWithDocs,
+  nodesRecallMarker,
   pollUntil,
   uniqueMarker,
   type KbFixture,
@@ -75,7 +76,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J3: 检索精修 (live, 自清理
       ];
       const baseline = await pollUntil(
         () => reporter.runStep("retrieve baseline", JOURNEY_J3_ROUTES, retrieveArgs),
-        (run) => run.exitCode === 0 && run.stdout.includes(marker),
+        (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
         { timeoutMs: 180_000, intervalMs: 15_000 },
       );
       expect(baseline.satisfied, `基线未召回标记词 ${marker}`).toBe(true);
diff --git a/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts b/packages/commands/tests/e2e/knowledge/journeys/j4-service-tuning.e2e.test.ts
@@ -1,13 +1,20 @@
-// J4 service tuning: draft (beta) usable → config change persisted → released version usable after deploy.
+// J4 service tuning: draft (beta) usable → config change persisted → released version carries the change after deploy.
 // create + service create (search, initial draft/beta) → search --agent-version beta recalls (hard)
-// → service update --description → service get asserts the change (hard) → deploy → released search recalls (hard).
+// → service update --description/--temperature (scalar merge path) → get (beta) asserts (hard)
+// → service update --config-file tweaking kb_search_configs (whole-replace path) → get (beta) asserts
+//   the nested change AND that the replace kept the scalar tuning (hard) → deploy
+// → get (released version) asserts both changes landed in the published version (hard) → released search recalls (hard).
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, test } from "vite-plus/test";
 import { isKbAdminE2EReady, parseStdoutJson } from "../../helpers.ts";
 import { JOURNEY_J4_ROUTES } from "../../topic-routes.ts";
 import {
   cleanupKbFixture,
   createJourneyReporter,
   createKbWithDocs,
+  nodesRecallMarker,
   patchSearchServiceRetrievalConfig,
   pollUntil,
   uniqueMarker,
@@ -17,7 +24,7 @@ import {
 describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自清理)", () => {
   const workspaceId = process.env.BAILIAN_WORKSPACE_ID!;
 
-  test("draft 可用 → update 落库 → deploy → 正式版可用", async () => {
+  test("draft 可用 → update 落库 → deploy → 正式版详情携带修改且可用", async () => {
     const reporter = createJourneyReporter(import.meta.url);
     const marker = uniqueMarker("j4");
     const fixture: Partial<KbFixture> = {};
@@ -71,25 +78,33 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自
       ];
       const betaPoll = await pollUntil(
         () => reporter.runStep("search (beta)", JOURNEY_J4_ROUTES, betaSearchArgs),
-        (run) => run.exitCode === 0 && run.stdout.includes(marker),
+        (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
         { timeoutMs: 180_000, intervalMs: 15_000 },
       );
       reporter.recordNote(`beta search 轮询 ${betaPoll.attempts} 次`);
       expect(betaPoll.satisfied, `search(beta) 未召回标记词 ${marker}`).toBe(true);
 
-      // 3) update the description (top-level scalar, valid for the search scene too) → get asserts persistence (hard)
+      // 3) tune the draft: description (top-level) + temperature (config-level scalar,
+      //    read-merge-write keeps the backfilled retrieval params) → get asserts both (hard)
       const newDescription = `journey j4 tuned at ${Date.now()}`;
-      const updateRun = await reporter.runStep("service update --description", JOURNEY_J4_ROUTES, [
-        "knowledge",
-        "service",
-        "update",
-        "--agent-id",
-        agentId,
-        "--description",
-        newDescription,
-        "--workspace-id",
-        workspaceId,
-      ]);
+      const tunedTemperature = 0.55;
+      const updateRun = await reporter.runStep(
+        "service update --description --temperature",
+        JOURNEY_J4_ROUTES,
+        [
+          "knowledge",
+          "service",
+          "update",
+          "--agent-id",
+          agentId,
+          "--description",
+          newDescription,
+          "--temperature",
+          String(tunedTemperature),
+          "--workspace-id",
+          workspaceId,
+        ],
+      );
       expect(updateRun.exitCode, updateRun.stderr).toBe(0);
 
       const getRun = await reporter.runStep("service get (beta)", JOURNEY_J4_ROUTES, [
@@ -106,10 +121,88 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自
         "json",
       ]);
       expect(getRun.exitCode, getRun.stderr).toBe(0);
-      const getData = parseStdoutJson<{ data?: { agent_desc?: string } }>(getRun.stdout);
+      const getData = parseStdoutJson<{
+        data?: {
+          agent_desc?: string;
+          agent_details?: Array<{
+            agent_config?: { temperature?: number } & Record<string, unknown>;
+          }>;
+        };
+      }>(getRun.stdout);
       expect(getData.data?.agent_desc).toBe(newDescription);
+      expect(getData.data?.agent_details?.[0]?.agent_config?.temperature).toBe(tunedTemperature);
+
+      // 3.5) complex nested tuning via --config-file (whole-replace path, distinct
+      //      from the scalar merge path above): read the current beta config, tweak
+      //      kb_search_configs, write it back — then assert the nested change landed
+      //      AND the replace kept the scalar tuning intact
+      const tunedDenseTopK = 66; // distinctive value, still recall-safe (loose top-k)
+      const betaConfig = getData.data?.agent_details?.[0]?.agent_config as
+        | ({ kb_search_configs?: Array<Record<string, unknown>> } & Record<string, unknown>)
+        | undefined;
+      expect(betaConfig?.kb_search_configs?.length, "beta 配置应含 kb_search_configs").toBeTruthy();
+      for (const kbConfig of betaConfig!.kb_search_configs!) {
+        kbConfig.dense_similarity_top_k = tunedDenseTopK;
+      }
+      const configDir = mkdtempSync(join(tmpdir(), "j4-config-"));
+      const configFile = join(configDir, "agent-config.json");
+      writeFileSync(configFile, JSON.stringify(betaConfig));
+      const configUpdateRun = await reporter.runStep(
+        "service update --config-file (kb_search_configs)",
+        JOURNEY_J4_ROUTES,
+        [
+          "knowledge",
+          "service",
+          "update",
+          "--agent-id",
+          agentId,
+          "--config-file",
+          configFile,
+          "--workspace-id",
+          workspaceId,
+        ],
+      );
+      expect(configUpdateRun.exitCode, configUpdateRun.stderr).toBe(0);
+
+      const betaAfterConfigRun = await reporter.runStep(
+        "service get (beta, after config-file)",
+        JOURNEY_J4_ROUTES,
+        [
+          "knowledge",
+          "service",
+          "get",
+          "--agent-id",
+          agentId,
+          "--agent-version",
+          "beta",
+          "--workspace-id",
+          workspaceId,
+          "--output",
+          "json",
+        ],
+      );
+      expect(betaAfterConfigRun.exitCode, betaAfterConfigRun.stderr).toBe(0);
+      const betaAfterConfig = parseStdoutJson<{
+        data?: {
+          agent_details?: Array<{
+            agent_config?: {
+              temperature?: number;
+              kb_search_configs?: Array<{ dense_similarity_top_k?: number }>;
+            };
+          }>;
+        };
+      }>(betaAfterConfigRun.stdout).data?.agent_details?.[0]?.agent_config;
+      expect(
+        betaAfterConfig?.kb_search_configs?.[0]?.dense_similarity_top_k,
+        "kb_search_configs 嵌套修改未落库",
+      ).toBe(tunedDenseTopK);
+      expect(betaAfterConfig?.temperature, "config-file 整体替换不应冲掉已调优的 temperature").toBe(
+        tunedTemperature,
+      );
 
-      // 4) deploy → released search (without --agent-version) recalls (hard)
+      // 4) deploy → the published version's detail must carry the tuned config (hard):
+      //    search alone only proves the released service responds, not that the
+      //    change actually shipped
       const deployRun = await reporter.runStep("service deploy", JOURNEY_J4_ROUTES, [
         "knowledge",
         "service",
@@ -122,8 +215,52 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自
         "--quiet",
       ]);
       expect(deployRun.exitCode, deployRun.stderr).toBe(0);
-      reporter.recordNote(`deploy 版本号: ${deployRun.stdout.trim().split("\n").pop() ?? "?"}`);
+      const deployedVersion = deployRun.stdout.trim().split("\n").pop() ?? "";
+      expect(deployedVersion, "deploy 应输出新版本号").toBeTruthy();
+      reporter.recordNote(`deploy 版本号: ${deployedVersion}`);
+
+      const releasedGetRun = await reporter.runStep(
+        `service get (released v${deployedVersion})`,
+        JOURNEY_J4_ROUTES,
+        [
+          "knowledge",
+          "service",
+          "get",
+          "--agent-id",
+          agentId,
+          "--agent-version",
+          deployedVersion,
+          "--workspace-id",
+          workspaceId,
+          "--output",
+          "json",
+        ],
+      );
+      expect(releasedGetRun.exitCode, releasedGetRun.stderr).toBe(0);
+      const releasedData = parseStdoutJson<{
+        data?: {
+          agent_details?: Array<{
+            agent_version?: string;
+            agent_config?: {
+              temperature?: number;
+              kb_search_configs?: Array<{ dense_similarity_top_k?: number }>;
+            };
+          }>;
+        };
+      }>(releasedGetRun.stdout);
+      const releasedDetail = releasedData.data?.agent_details?.find(
+        (detail) => detail.agent_version === deployedVersion,
+      );
+      expect(releasedDetail, `get 应返回已发布版本 ${deployedVersion} 的详情`).toBeTruthy();
+      expect(releasedDetail?.agent_config?.temperature, "调优的 temperature 未进入正式版配置").toBe(
+        tunedTemperature,
+      );
+      expect(
+        releasedDetail?.agent_config?.kb_search_configs?.[0]?.dense_similarity_top_k,
+        "调优的 kb_search_configs 未进入正式版配置",
+      ).toBe(tunedDenseTopK);
 
+      // 5) released search (without --agent-version) recalls (hard)
       const releasedPoll = await pollUntil(
         () =>
           reporter.runStep("search (released)", JOURNEY_J4_ROUTES, [
@@ -138,7 +275,7 @@ describe.skipIf(!isKbAdminE2EReady())("journey J4: 问答服务调优 (live, 自
             "--output",
             "json",
           ]),
-        (run) => run.exitCode === 0 && run.stdout.includes(marker),
+        (run) => run.exitCode === 0 && nodesRecallMarker(run.stdout, marker),
         { timeoutMs: 120_000, intervalMs: 15_000 },
       );
       reporter.recordNote(`正式版 search 轮询 ${releasedPoll.attempts} 次`);
diff --git a/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts b/packages/commands/tests/e2e/knowledge/journeys/journey-helpers.ts
@@ -377,6 +377,29 @@ export async function cleanupKbFixture(
   }
 }
 
+// ---- Marker-recall predicate (strict) ----
+
+/**
+ * Positive recall check for retrieve/search JSON output: the marker must appear
+ * inside a returned node's text (not merely anywhere in the raw output — doc_name
+ * or other metadata echoing the marker must not count as recall).
+ * Negative assertions ("marker gone") should keep scanning the whole stdout:
+ * absence from the entire output is the stronger guarantee.
+ */
+export function nodesRecallMarker(stdout: string, marker: string): boolean {
+  try {
+    const payload = JSON.parse(stdout) as {
+      data?: { nodes?: Array<{ text?: string; metadata?: { content?: string } }> };
+    };
+    return (payload.data?.nodes ?? []).some((node) =>
+      (node.metadata?.content ?? node.text ?? "").includes(marker),
+    );
+  } catch {
+    // Mid-poll runs may fail before emitting JSON — treat as "not recalled yet"
+    return false;
+  }
+}
+
 // ---- Backfill required retrieval parameters for search services (server gotcha) ----
 // The minimal kb_search_configs created by service create --index-id is missing
 // required fields like rerank_min_score / dense_similarity_top_k, so the search
