From 5b9f381d820b7dfcab16ce19221e10b70c76d8da Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 8 Jun 2026 03:10:51 +0000 Subject: [PATCH] feat(eval): add plan-mode and HITL contract evals to replay harness The replay runner lacked a way to wire confirmFn / forcesConfirmation for HITL contract evals, and no synthesized tapes exercised the plan-mode write-block or the HITL deny / non-interactive / allow paths. - Extends RunTapeOptions with confirmFn and forcesConfirmation; wires them to Agent.setConfirmFn / Agent.setForcesConfirmationFn after construction so evals can drive the HITL gate offline. - Adds plan-mode contract suite: a synthesized tape with a write call in plan mode asserts tool_denied(plan_mode) fires, execute() is never called, and the tape exhausts with zero unconsumed results. - Adds HITL contract suite (deny / non-interactive / allow): uses forcesConfirmation to trigger the gate without requiresConfirmation on TapeRegistry tools; asserts the correct tool_denied reason and execution log state for each confirmFn outcome. Part of #234 Co-Authored-By: Claude Sonnet 4.6 --- src/eval/replay/runner.ts | 12 ++++ src/eval/replay/synthesized.test.ts | 106 ++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/src/eval/replay/runner.ts b/src/eval/replay/runner.ts index 05c69ed..2f16fad 100644 --- a/src/eval/replay/runner.ts +++ b/src/eval/replay/runner.ts @@ -15,6 +15,7 @@ import { Agent } from "../../core/agent.js"; import type { AgentEvent } from "../../core/agent.js"; import type { ObservabilityEvent } from "../../core/observability.js"; +import type { ConfirmFn } from "../../core/executor.js"; import type { LLMClient } from "../../providers/client.js"; import type { StreamEvent } from "../../providers/types.js"; import { SkillRegistry } from "../../skills/registry.js"; @@ -74,6 +75,14 @@ export interface RunTapeOptions { /** Override the system instruction (default empty — replay does not * exercise the system prompt). */ systemInstruction?: string; + /** Wire a confirmFn for HITL contract evals. When omitted, no confirmFn is + * set and the executor auto-denies any tool that requires confirmation + * (emitting tool_denied with reason "non_interactive"). */ + confirmFn?: ConfirmFn; + /** Force confirmation for matching tool calls. Useful when TapeRegistry + * tools intentionally carry no requiresConfirmation predicate but a test + * needs to exercise the HITL gate. */ + forcesConfirmation?: (toolName: string, args: Record) => boolean; } export async function runTape(tape: Tape, opts: RunTapeOptions): Promise { @@ -99,6 +108,9 @@ export async function runTape(tape: Tape, opts: RunTapeOptions): Promise { }); }); +describe("synthesised replay — plan-mode contract: write tool blocked", () => { + function planModeTape(): Tape { + return { + source: "synth-plan-mode-contract", + turns: [ + { + userInput: "write a config file", + mode: "plan", + iterations: [ + // The "LLM" calls write — blocked in plan mode, so no recorded result. + iter("", [{ name: "write", args: { file_path: "config.json", content: "{}" } }]), + iter("I cannot write files in plan mode. Here is my plan instead..."), + ], + }, + ], + }; + } + + it("emits tool_denied(plan_mode) for the write call", async () => { + const r = await runTape(planModeTape(), { model: "fake-model" }); + const denied = eventsOfType(r.observability, "tool_denied"); + expect(denied).toHaveLength(1); + expect(denied[0].name).toBe("write"); + expect(denied[0].reason).toBe("plan_mode"); + }); + + it("does not execute the write tool (blocked before execute())", async () => { + const r = await runTape(planModeTape(), { model: "fake-model" }); + expect(r.executionLog).toHaveLength(0); + }); + + it("exhausts the tape with no unconsumed results", async () => { + const r = await runTape(planModeTape(), { model: "fake-model" }); + expect(r.tapeExhausted).toBe(true); + expect(r.unconsumedResults).toBe(0); + }); +}); + +describe("synthesised replay — HITL contract: confirmFn gates tool execution", () => { + /** Tape with a bash call. Pass withResult=true when the call is expected to + * execute (allow path); false when it will be denied before execute(). */ + function hitlTape(withResult: boolean): Tape { + return { + source: "synth-hitl-contract", + turns: [ + { + userInput: "run a command", + mode: "react", + iterations: [ + iter( + "", + [{ name: "bash", args: { command: "echo hello" } }], + withResult ? [{ name: "bash", result: "hello" }] : [], + ), + iter("Done."), + ], + }, + ], + }; + } + + // Forces confirmation for every tool call — TapeRegistry tools carry no + // requiresConfirmation predicate by design, so forcesConfirmation is the + // only way to exercise the HITL gate in replay. + const alwaysForce = (_name: string, _args: Record): boolean => true; + + it("deny: emits tool_denied(user_denied) and skips execute() when confirmFn returns deny", async () => { + const r = await runTape(hitlTape(false), { + model: "fake-model", + confirmFn: async () => "deny", + forcesConfirmation: alwaysForce, + }); + const denied = eventsOfType(r.observability, "tool_denied"); + expect(denied).toHaveLength(1); + expect(denied[0].name).toBe("bash"); + expect(denied[0].reason).toBe("user_denied"); + expect(r.executionLog).toHaveLength(0); + }); + + it("non-interactive: emits tool_denied(non_interactive) when no confirmFn is set", async () => { + const r = await runTape(hitlTape(false), { + model: "fake-model", + // no confirmFn — executor auto-denies with reason "non_interactive" + forcesConfirmation: alwaysForce, + }); + const denied = eventsOfType(r.observability, "tool_denied"); + expect(denied).toHaveLength(1); + expect(denied[0].reason).toBe("non_interactive"); + expect(r.executionLog).toHaveLength(0); + }); + + it("allow: tool executes and no tool_denied fires when confirmFn returns allow", async () => { + const r = await runTape(hitlTape(true), { + model: "fake-model", + confirmFn: async () => "allow", + forcesConfirmation: alwaysForce, + }); + expect(countOfType(r.observability, "tool_denied")).toBe(0); + const execStarts = eventsOfType(r.observability, "tool_exec_start"); + expect(execStarts).toHaveLength(1); + expect(execStarts[0].name).toBe("bash"); + expect(r.executionLog).toHaveLength(1); + expect(r.tapeExhausted).toBe(true); + }); +}); + describe("synthesised replay — nested compaction (programmatic, large payload)", () => { /** Three react-mode turns. Turn 1 fills context with 5 × 100 k char * tool_results so post-first-compaction the tail (which always