From 730aa0d6883a3a5935f3f03fcc59e5594ab78f39 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:11:12 +0200 Subject: [PATCH 1/5] fix CL-01 negative-control accounting --- src/lab/conformance/runner.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts index 864f887266..a28c924307 100644 --- a/src/lab/conformance/runner.ts +++ b/src/lab/conformance/runner.ts @@ -1,7 +1,7 @@ import { discoverScenarios, loadCaseAuthority } from "./manifest"; import { runScenario } from "./executor"; import { buildNegativeControls } from "./negative-controls"; -import type { ScenarioRunResult } from "./types"; +import type { CaseRecord, ScenarioRunResult } from "./types"; import { CL01_SUITES } from "./types"; export interface ConformanceRunSummary { @@ -19,6 +19,8 @@ export interface NegativeControlRunSummary extends ConformanceRunSummary { rejected: number; } +type ScenarioRunner = (caseRecord: CaseRecord) => Promise; + export async function runConformanceSuite( suites: readonly string[] = CL01_SUITES, ): Promise { @@ -33,15 +35,21 @@ export async function runConformanceSuite( return { total: results.length, passed, failed: results.length - passed, results }; } -export async function runNegativeControls(): Promise { +export async function runNegativeControls( + execute: ScenarioRunner = runScenario, +): Promise { const authority = loadCaseAuthority(); const scenarios = buildNegativeControls(discoverScenarios(authority)); if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { - results.push(await runScenario(scenario)); + results.push(await execute(scenario)); } - const rejected = results.filter((r) => !r.passed).length; + const rejected = results.filter((r) => ( + !r.passed + && r.classification === "protocol_failure" + && r.secondaryCode === "deterministic_assertion" + )).length; return { total: results.length, passed: rejected, From 3ecd63fa0b6df30f36d8550911e6a25163f80058 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:11:23 +0200 Subject: [PATCH 2/5] test CL-01 harness failure isolation --- tests/lab-conformance-runner-failures.test.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/lab-conformance-runner-failures.test.ts diff --git a/tests/lab-conformance-runner-failures.test.ts b/tests/lab-conformance-runner-failures.test.ts new file mode 100644 index 0000000000..52414b6ed5 --- /dev/null +++ b/tests/lab-conformance-runner-failures.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "bun:test"; +import { runScenario } from "../src/lab/conformance/executor"; +import { NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; +import { runNegativeControls } from "../src/lab/conformance/runner"; + +describe("CL-01 negative-control failure accounting", () => { + test("does not count harness failures as rejected negative controls", async () => { + let injected = false; + const summary = await runNegativeControls(async (scenario) => { + const result = await runScenario(scenario); + if (injected) return result; + injected = true; + return { + ...result, + passed: false, + classification: "harness_failure", + secondaryCode: "execution_error", + assertionResults: [], + diagnostics: ["synthetic harness failure"], + }; + }); + + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(summary.rejected).toBe(summary.total - 1); + expect(summary.passed).toBe(summary.rejected); + expect(summary.failed).toBe(1); + expect(summary.results.some((result) => result.classification === "harness_failure")).toBe(true); + }, 120000); + + test("counts deterministic protocol failures as rejected negative controls", async () => { + const summary = await runNegativeControls(); + + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(summary.rejected).toBe(summary.total); + expect(summary.failed).toBe(0); + for (const result of summary.results) { + expect(result.passed).toBe(false); + expect(result.classification).toBe("protocol_failure"); + expect(result.secondaryCode).toBe("deterministic_assertion"); + } + }, 120000); +}); From de9b5ff46c1ef6414d8b84a4180f5079d9411c6f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:23:24 +0200 Subject: [PATCH 3/5] fix CL-01 negative-control execution identity --- src/lab/conformance/runner.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts index a28c924307..cc8bdcc9e8 100644 --- a/src/lab/conformance/runner.ts +++ b/src/lab/conformance/runner.ts @@ -1,6 +1,6 @@ import { discoverScenarios, loadCaseAuthority } from "./manifest"; import { runScenario } from "./executor"; -import { buildNegativeControls } from "./negative-controls"; +import { baseCaseForNegativeControl, buildNegativeControls } from "./negative-controls"; import type { CaseRecord, ScenarioRunResult } from "./types"; import { CL01_SUITES } from "./types"; @@ -43,7 +43,10 @@ export async function runNegativeControls( if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { - results.push(await execute(scenario)); + const baseCase = baseCaseForNegativeControl(scenario.id, authority.cases); + const executionScenario = baseCase ? { ...scenario, id: baseCase.id } : scenario; + const result = await execute(executionScenario); + results.push({ ...result, scenarioId: scenario.id }); } const rejected = results.filter((r) => ( !r.passed From 28cf62aa97af1a3b0dbe8dcc33fb909702866023 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:28:00 +0200 Subject: [PATCH 4/5] test: report unexpected CL-01 negative-control classifications --- tests/lab-conformance-runner-failures.test.ts | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/lab-conformance-runner-failures.test.ts b/tests/lab-conformance-runner-failures.test.ts index 52414b6ed5..47fc707bd9 100644 --- a/tests/lab-conformance-runner-failures.test.ts +++ b/tests/lab-conformance-runner-failures.test.ts @@ -3,6 +3,21 @@ import { runScenario } from "../src/lab/conformance/executor"; import { NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; import { runNegativeControls } from "../src/lab/conformance/runner"; +function unexpectedNegativeControlResults( + results: Awaited>["results"], +): string[] { + return results + .filter((result) => ( + result.passed + || result.classification !== "protocol_failure" + || result.secondaryCode !== "deterministic_assertion" + )) + .map((result) => ( + `${result.scenarioId}: ${result.classification}/${result.secondaryCode ?? "none"}` + + (result.diagnostics.length > 0 ? ` ${result.diagnostics.join(";")}` : "") + )); +} + describe("CL-01 negative-control failure accounting", () => { test("does not count harness failures as rejected negative controls", async () => { let injected = false; @@ -24,19 +39,15 @@ describe("CL-01 negative-control failure accounting", () => { expect(summary.rejected).toBe(summary.total - 1); expect(summary.passed).toBe(summary.rejected); expect(summary.failed).toBe(1); - expect(summary.results.some((result) => result.classification === "harness_failure")).toBe(true); + expect(summary.results.filter((result) => result.classification === "harness_failure")).toHaveLength(1); }, 120000); test("counts deterministic protocol failures as rejected negative controls", async () => { const summary = await runNegativeControls(); expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(unexpectedNegativeControlResults(summary.results)).toEqual([]); expect(summary.rejected).toBe(summary.total); expect(summary.failed).toBe(0); - for (const result of summary.results) { - expect(result.passed).toBe(false); - expect(result.classification).toBe("protocol_failure"); - expect(result.secondaryCode).toBe("deterministic_assertion"); - } }, 120000); }); From 1c59fb18d307266548d376d2fdc9064bd954f01c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:33:08 +0200 Subject: [PATCH 5/5] fix CL-01 tool-result negative control --- src/lab/conformance/negative-controls.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts index 66ab70f4e7..3391e9669a 100644 --- a/src/lab/conformance/negative-controls.ts +++ b/src/lab/conformance/negative-controls.ts @@ -50,11 +50,15 @@ export const NEGATIVE_CONTROL_FIXTURES: Array<{ }, { id: "negative.tool-result-order", - defect: "invalid tool-result ordering", + defect: "invalid tool-result correlation after chat-history repair", mutate: (c) => ({ ...c, id: "negative.tool-result-order", - assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/1/tool_call_id" }, required: true }], + // Chat history hardening closes the unmatched call with a synthetic result, then + // re-emits the orphan result behind a synthetic assistant call. Inspect the actual + // supplied result at the end of that repaired pair rather than the synthetic close, + // otherwise the negative control accidentally validates the repair and passes. + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/3/tool_call_id" }, required: true }], fixture: { ...c.fixture, bytesUtf8: JSON.stringify({