diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts index 66ab70f4e7..3391e9669a 100644 --- a/src/lab/conformance/negative-controls.ts +++ b/src/lab/conformance/negative-controls.ts @@ -50,11 +50,15 @@ export const NEGATIVE_CONTROL_FIXTURES: Array<{ }, { id: "negative.tool-result-order", - defect: "invalid tool-result ordering", + defect: "invalid tool-result correlation after chat-history repair", mutate: (c) => ({ ...c, id: "negative.tool-result-order", - assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/1/tool_call_id" }, required: true }], + // Chat history hardening closes the unmatched call with a synthetic result, then + // re-emits the orphan result behind a synthetic assistant call. Inspect the actual + // supplied result at the end of that repaired pair rather than the synthetic close, + // otherwise the negative control accidentally validates the repair and passes. + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/3/tool_call_id" }, required: true }], fixture: { ...c.fixture, bytesUtf8: JSON.stringify({ diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts index 864f887266..cc8bdcc9e8 100644 --- a/src/lab/conformance/runner.ts +++ b/src/lab/conformance/runner.ts @@ -1,7 +1,7 @@ import { discoverScenarios, loadCaseAuthority } from "./manifest"; import { runScenario } from "./executor"; -import { buildNegativeControls } from "./negative-controls"; -import type { ScenarioRunResult } from "./types"; +import { baseCaseForNegativeControl, buildNegativeControls } from "./negative-controls"; +import type { CaseRecord, ScenarioRunResult } from "./types"; import { CL01_SUITES } from "./types"; export interface ConformanceRunSummary { @@ -19,6 +19,8 @@ export interface NegativeControlRunSummary extends ConformanceRunSummary { rejected: number; } +type ScenarioRunner = (caseRecord: CaseRecord) => Promise; + export async function runConformanceSuite( suites: readonly string[] = CL01_SUITES, ): Promise { @@ -33,15 +35,24 @@ export async function runConformanceSuite( return { total: results.length, passed, failed: results.length - passed, results }; } -export async function runNegativeControls(): Promise { +export async function runNegativeControls( + execute: ScenarioRunner = runScenario, +): Promise { const authority = loadCaseAuthority(); const scenarios = buildNegativeControls(discoverScenarios(authority)); if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { - results.push(await runScenario(scenario)); + const baseCase = baseCaseForNegativeControl(scenario.id, authority.cases); + const executionScenario = baseCase ? { ...scenario, id: baseCase.id } : scenario; + const result = await execute(executionScenario); + results.push({ ...result, scenarioId: scenario.id }); } - const rejected = results.filter((r) => !r.passed).length; + const rejected = results.filter((r) => ( + !r.passed + && r.classification === "protocol_failure" + && r.secondaryCode === "deterministic_assertion" + )).length; return { total: results.length, passed: rejected, diff --git a/tests/lab-conformance-runner-failures.test.ts b/tests/lab-conformance-runner-failures.test.ts new file mode 100644 index 0000000000..47fc707bd9 --- /dev/null +++ b/tests/lab-conformance-runner-failures.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from "bun:test"; +import { runScenario } from "../src/lab/conformance/executor"; +import { NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; +import { runNegativeControls } from "../src/lab/conformance/runner"; + +function unexpectedNegativeControlResults( + results: Awaited>["results"], +): string[] { + return results + .filter((result) => ( + result.passed + || result.classification !== "protocol_failure" + || result.secondaryCode !== "deterministic_assertion" + )) + .map((result) => ( + `${result.scenarioId}: ${result.classification}/${result.secondaryCode ?? "none"}` + + (result.diagnostics.length > 0 ? ` ${result.diagnostics.join(";")}` : "") + )); +} + +describe("CL-01 negative-control failure accounting", () => { + test("does not count harness failures as rejected negative controls", async () => { + let injected = false; + const summary = await runNegativeControls(async (scenario) => { + const result = await runScenario(scenario); + if (injected) return result; + injected = true; + return { + ...result, + passed: false, + classification: "harness_failure", + secondaryCode: "execution_error", + assertionResults: [], + diagnostics: ["synthetic harness failure"], + }; + }); + + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(summary.rejected).toBe(summary.total - 1); + expect(summary.passed).toBe(summary.rejected); + expect(summary.failed).toBe(1); + expect(summary.results.filter((result) => result.classification === "harness_failure")).toHaveLength(1); + }, 120000); + + test("counts deterministic protocol failures as rejected negative controls", async () => { + const summary = await runNegativeControls(); + + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(unexpectedNegativeControlResults(summary.results)).toEqual([]); + expect(summary.rejected).toBe(summary.total); + expect(summary.failed).toBe(0); + }, 120000); +});