fix(lab): isolate CL-01 harness failures from negative-control passes

src/lab/conformance/negative-controls.ts

@@ -50,11 +50,15 @@ export const NEGATIVE_CONTROL_FIXTURES: Array<{
   },
   {
     id: "negative.tool-result-order",
-    defect: "invalid tool-result ordering",
+    defect: "invalid tool-result correlation after chat-history repair",
     mutate: (c) => ({
       ...c,
       id: "negative.tool-result-order",
-      assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/1/tool_call_id" }, required: true }],
+      // Chat history hardening closes the unmatched call with a synthetic result, then
+      // re-emits the orphan result behind a synthetic assistant call. Inspect the actual
+      // supplied result at the end of that repaired pair rather than the synthetic close,
+      // otherwise the negative control accidentally validates the repair and passes.
+      assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/3/tool_call_id" }, required: true }],
       fixture: {
         ...c.fixture,
         bytesUtf8: JSON.stringify({

src/lab/conformance/runner.ts

@@ -1,7 +1,7 @@
 import { discoverScenarios, loadCaseAuthority } from "./manifest";
 import { runScenario } from "./executor";
-import { buildNegativeControls } from "./negative-controls";
-import type { ScenarioRunResult } from "./types";
+import { baseCaseForNegativeControl, buildNegativeControls } from "./negative-controls";
+import type { CaseRecord, ScenarioRunResult } from "./types";
 import { CL01_SUITES } from "./types";
 
 export interface ConformanceRunSummary {
@@ -19,6 +19,8 @@ export interface NegativeControlRunSummary extends ConformanceRunSummary {
   rejected: number;
 }
 
+type ScenarioRunner = (caseRecord: CaseRecord) => Promise<ScenarioRunResult>;
+
 export async function runConformanceSuite(
   suites: readonly string[] = CL01_SUITES,
 ): Promise<ConformanceRunSummary> {
@@ -33,15 +35,24 @@ export async function runConformanceSuite(
   return { total: results.length, passed, failed: results.length - passed, results };
 }
 
-export async function runNegativeControls(): Promise<NegativeControlRunSummary> {
+export async function runNegativeControls(
+  execute: ScenarioRunner = runScenario,
+): Promise<NegativeControlRunSummary> {
   const authority = loadCaseAuthority();
   const scenarios = buildNegativeControls(discoverScenarios(authority));
   if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered");
   const results: ScenarioRunResult[] = [];
   for (const scenario of scenarios) {
-    results.push(await runScenario(scenario));
+    const baseCase = baseCaseForNegativeControl(scenario.id, authority.cases);
+    const executionScenario = baseCase ? { ...scenario, id: baseCase.id } : scenario;
+    const result = await execute(executionScenario);
+    results.push({ ...result, scenarioId: scenario.id });
   }
-  const rejected = results.filter((r) => !r.passed).length;
+  const rejected = results.filter((r) => (
+    !r.passed
+    && r.classification === "protocol_failure"
+    && r.secondaryCode === "deterministic_assertion"
+  )).length;
   return {
     total: results.length,
     passed: rejected,

tests/lab-conformance-runner-failures.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "bun:test";
+import { runScenario } from "../src/lab/conformance/executor";
+import { NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls";
+import { runNegativeControls } from "../src/lab/conformance/runner";
+
+function unexpectedNegativeControlResults(
+  results: Awaited<ReturnType<typeof runNegativeControls>>["results"],
+): string[] {
+  return results
+    .filter((result) => (
+      result.passed
+      || result.classification !== "protocol_failure"
+      || result.secondaryCode !== "deterministic_assertion"
+    ))
+    .map((result) => (
+      `${result.scenarioId}: ${result.classification}/${result.secondaryCode ?? "none"}`
+      + (result.diagnostics.length > 0 ? ` ${result.diagnostics.join(";")}` : "")
+    ));
+}
+
+describe("CL-01 negative-control failure accounting", () => {
+  test("does not count harness failures as rejected negative controls", async () => {
+    let injected = false;
+    const summary = await runNegativeControls(async (scenario) => {
+      const result = await runScenario(scenario);
+      if (injected) return result;
+      injected = true;
+      return {
+        ...result,
+        passed: false,
+        classification: "harness_failure",
+        secondaryCode: "execution_error",
+        assertionResults: [],
+        diagnostics: ["synthetic harness failure"],
+      };
+    });
+
+    expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length);
+    expect(summary.rejected).toBe(summary.total - 1);
+    expect(summary.passed).toBe(summary.rejected);
+    expect(summary.failed).toBe(1);
+    expect(summary.results.filter((result) => result.classification === "harness_failure")).toHaveLength(1);
+  }, 120000);
+
+  test("counts deterministic protocol failures as rejected negative controls", async () => {
+    const summary = await runNegativeControls();
+
+    expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length);
+    expect(unexpectedNegativeControlResults(summary.results)).toEqual([]);
+    expect(summary.rejected).toBe(summary.total);
+    expect(summary.failed).toBe(0);
+  }, 120000);
+});