From 71451943aaca4aaf2d1f006aee951482588e3b91 Mon Sep 17 00:00:00 2001 From: even-wei Date: Fri, 29 May 2026 17:34:12 +0800 Subject: [PATCH 1/4] feat(evals): Karpathy spike driver for /recce-verify v1 eval (DRC-3586) Single-file Python driver at evals/agent-blind-spots/spike-driver/. Dispatches up to 6 fixtures x 2 agents x 2 tiers = 24 cells, captures each agent's transcript, runs a Claude-as-judge pass per transcript to produce a three-axis verdict (catch / tier / delta), and writes a CSV + Markdown summary under runs//spike-driver/. Stability checks (both optional, both produce judge-quality signal): - --judge-stability: double-judges each transcript, reports per-axis self-consistency (catch / tier / delta). Floor 80%. - --baseline-dir : compares judge verdicts against DRC-3585 manual baseline once it lands (judge-vs-human catch agreement on Tier-0 cells). Graceful degradation: - Skips an agent's cells if its CLI is not on PATH; codex commonly absent in lighter dev envs (recipe in runner-configs/codex/tier-{0,1}/README.md). - --no-run mode re-judges existing transcripts without re-running agents. Sandbox profile integration (from DRC-3584): - Claude Code cells stamp tier-N claude-overlay/ into the fixture worktree before invoking `claude --print`, with a neutered CLAUDE_CONFIG_DIR per cell and scrubbed warehouse env. - Codex cells use `codex exec --sandbox= --ask-for-approval=never --config ` plus PATH scrub. Non-goals for this spike (deferred): - Durable harness with resume / parallel dispatch -> DRC-3587 Inspect AI port. - Gap-report generator across 6 fixtures -> DRC-3405. - Auto-iteration on prompt / skill changes (closed-loop optimization overfits at N=6 -- explicit non-goal of the spike). Stacks on PR #36 (DRC-3584 + DRC-3430). When #36 merges to main, this PR should be rebased onto main; GitHub auto-retargets the base branch. Signed-off-by: even-wei --- .../agent-blind-spots/spike-driver/README.md | 80 ++++ .../agent-blind-spots/spike-driver/driver.py | 415 ++++++++++++++++++ 2 files changed, 495 insertions(+) create mode 100644 evals/agent-blind-spots/spike-driver/README.md create mode 100644 evals/agent-blind-spots/spike-driver/driver.py diff --git a/evals/agent-blind-spots/spike-driver/README.md b/evals/agent-blind-spots/spike-driver/README.md new file mode 100644 index 0000000..eea3060 --- /dev/null +++ b/evals/agent-blind-spots/spike-driver/README.md @@ -0,0 +1,80 @@ +# Spike driver — DRC-3586 + +Single-file Python driver for the Karpathy-style spike in [DRC-3586](https://linear.app/recce/issue/DRC-3586). De-risks two unknowns before any durable-harness work (DRC-3587): + +1. **Judge stability** — does the Claude-as-judge prompt produce consistent verdicts on the locked three-axis rubric? +2. **Codex-under-sandbox** — does Codex behave in a programmatic loop under the Tier-0 / Tier-1 sandbox profiles from DRC-3584? + +Not the durable harness. Output artifacts live under `runs//spike-driver/`. If the spike's stability + sandbox checks pass, the next ticket (DRC-3587) ports this to Inspect AI; if they fail, the project falls back to Candidate B (manual scoring loop). + +## Prereqs + +- Per-fixture worktrees built: `( cd evals/agent-blind-spots && ./build_fixtures.sh )` +- `bashlex` installed for the Claude Code Tier-0/1 PreToolUse hook: `python3 -m pip install bashlex` +- `claude` CLI on PATH (for both agent + judge passes) +- `codex` CLI on PATH (optional — codex cells skip gracefully if missing) +- `uv` available (per repo convention) + +## Usage + +```bash +# Smoke test (1 fixture × 2 agents × 2 tiers = 4 cells, ~5-10 min) +uv run evals/agent-blind-spots/spike-driver/driver.py --smoke + +# Full run (6 fixtures × 2 agents × 2 tiers = 24 cells, ~1-2 hours) +uv run evals/agent-blind-spots/spike-driver/driver.py + +# Limit to one agent / tier +uv run evals/agent-blind-spots/spike-driver/driver.py --agents claude --tiers 0 + +# Double-judge each transcript for self-consistency +uv run evals/agent-blind-spots/spike-driver/driver.py --judge-stability + +# Re-judge an existing run without re-running agents +uv run evals/agent-blind-spots/spike-driver/driver.py \ + --no-run --run-dir evals/agent-blind-spots/runs/2026-05-29/spike-driver/ + +# Compare judge verdicts to DRC-3585 manual baseline once it lands +uv run evals/agent-blind-spots/spike-driver/driver.py \ + --baseline-dir evals/agent-blind-spots/fixtures/ +``` + +## Output layout + +``` +runs//spike-driver/ +├── transcripts/ # raw agent stdout/stderr per cell +│ ├── pr1-fix-clv_claude_t0.txt +│ ├── pr1-fix-clv_claude_t1.txt +│ ├── pr1-fix-clv_codex_t0.txt +│ └── ... +├── _claude_cfg/ # neutered CLAUDE_CONFIG_DIR per cell +├── verdicts.csv # one row per cell +├── cells.json # raw cells (re-judgeable) +└── summary.md # rendered matrix + stability check +``` + +## Stability checks + +Two ways to validate judge quality (use both when possible): + +**Self-consistency** (`--judge-stability`): judges each transcript twice in independent subprocess calls; reports per-axis agreement. Floor: ≥80% on each of catch / tier / delta. Below that, the judge is too noisy to replace hand-grading. + +**Judge vs human** (`--baseline-dir `): if [DRC-3585](https://linear.app/recce/issue/DRC-3585)'s manual rubric-lock has produced `fixtures//tier-0-baseline.md` files, this compares the judge's catch verdict for Tier-0 cells to the human's. This is the canonical check from DRC-3586's acceptance criterion 2; self-consistency is the proxy when no manual baseline exists yet. + +## Codex-under-sandbox check + +After a run, grep transcripts under `transcripts/*_codex_*.txt` for: + +- `mcp__recce__*` references in Tier-0 → leak (MCP table is empty in `runner-configs/codex/tier-0/config.toml`). +- `recce ` shell calls in Tier-0 → check exit was non-zero (PATH scrub). +- Reads of `../README.md` or `../../RUBRIC.md` → leak (cwd separation). +- `dbt ` shell calls in either tier → check exit was non-zero (PATH scrub). + +If every Codex cell ran to completion without these escape attempts succeeding, criterion 3 passes. + +## Non-goals (deferred to DRC-3587 / DRC-3405) + +- Durable harness with resume / parallel dispatch — that's Inspect AI. +- Gap-report generator across 6 fixtures — that's DRC-3405. +- Auto-iteration on prompt / skill changes — explicit non-goal of the spike (closed-loop optimization overfits at N=6). diff --git a/evals/agent-blind-spots/spike-driver/driver.py b/evals/agent-blind-spots/spike-driver/driver.py new file mode 100644 index 0000000..c4c2067 --- /dev/null +++ b/evals/agent-blind-spots/spike-driver/driver.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Karpathy-style spike driver for /recce-verify v1 eval (DRC-3586). + +Dispatches up to 6 fixtures × 2 agents × 2 tiers = 24 cells, captures +each agent's transcript, runs a Claude-as-judge pass per transcript to +produce a three-axis verdict (catch / tier / delta), and writes a +CSV + Markdown summary under `runs//spike-driver/`. + +Two stability modes: + --judge-stability Run the judge twice per cell; report self-consistency. + --baseline-dir Compare judge verdicts against DRC-3585 manual baseline. + +Both are optional. Without either, the driver still produces a verdict +matrix; stability checks are how you decide whether the judge can replace +hand-grading at N=6. + +Usage: + uv run driver.py --smoke # 1 fixture × 2 agents × 2 tiers + uv run driver.py # full 24-cell run + uv run driver.py --agents claude # claude only + uv run driver.py --tiers 0 # tier-0 only + uv run driver.py --judge-stability # double-judge for self-consistency + uv run driver.py --no-run \\ + --run-dir runs/2026-05-29/spike-driver/ # re-judge only +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +SPIKE_DIR = Path(__file__).resolve().parent +EVAL_DIR = SPIKE_DIR.parent +SOURCES_DIR = EVAL_DIR / ".tmp" / "sources" +RUNS_DIR = EVAL_DIR / "runs" +RUNNER_CONFIGS = EVAL_DIR / "runner-configs" + +DEFAULT_FIXTURES = ( + "pr1-fix-clv", + "pr2-refactor-cte-to-models", + "pr3-amount-double-to-decimal", + "pr42-is-closed-filter", + "pr44-promotion-flags", + "pr46-net-clv-segments", +) +AGENTS = ("claude", "codex") +TIERS = (0, 1) + +AGENT_PROMPT = ( + "Review this dbt PR. The inputs listed in the Tier-0 runtime contract " + "(diff.patch, manifest-before.json / manifest-after.json, compiled-before/ / " + "compiled-after/, catalog-before.json / catalog-after.json, and read access " + "to the dbt project at the head SHA) are available in the current working " + "directory. Decide catch / miss / partial per the rubric. Recommend " + "approve / request-changes / abstain. End your output with one line, " + "exactly:\n" + "VERDICT: · " +) + +JUDGE_SYSTEM = ( + "You are a strict, terse judge. Score the agent transcript against the " + "locked three-axis rubric (catch, primary evidence tier, counterfactual " + "delta vs Tier-0 baseline). Emit ONLY JSON. No prose outside JSON." +) + +JUDGE_USER_TEMPLATE = """Rubric (excerpt): +{rubric} + +Fixture: {fixture} +Agent: {agent} +Tier: {tier} +Tier-0 baseline catch for this fixture (if known): {baseline_catch} + +Agent transcript: +--- +{transcript} +--- + +Emit JSON ONLY in this exact shape: +{{ + "catch": "catch" | "miss" | "partial", + "tier": "0" | "1a" | "1b" | "1c" | "2", + "delta": "improvement" | "same" | "regression", + "reasoning": "1-2 sentences citing the decisive evidence the agent used" +}} +""" + +VERDICT_TAIL_RE = re.compile(r"VERDICT:\s*(catch|miss|partial)\s*[·•|]?\s*(approve|request-changes|abstain)", re.IGNORECASE) + + +@dataclass +class Cell: + fixture: str + agent: str + tier: int + transcript_path: str | None = None + returncode: int | None = None + verdict: dict | None = None + verdict_2: dict | None = None + error: str | None = None + + +def cli_available(name: str) -> bool: + return shutil.which(name) is not None + + +def scrub_env(extra_unset: tuple[str, ...] = ()) -> dict: + env = os.environ.copy() + for var in ( + "RECCE_API_TOKEN", "DBT_PROFILES_DIR", + "SNOWFLAKE_USER", "SNOWFLAKE_PASSWORD", "SNOWFLAKE_ACCOUNT", + "POSTGRES_PASSWORD", "BIGQUERY_PROJECT", *extra_unset, + ): + env.pop(var, None) + return env + + +def scrub_path(strip_patterns: tuple[str, ...]) -> str: + pat = re.compile("|".join(strip_patterns)) + return ":".join(p for p in os.environ.get("PATH", "").split(":") if not pat.search(p)) + + +def run_claude(fixture_dir: Path, tier: int, run_dir: Path, prompt: str) -> tuple[Path, int]: + tier_dir = RUNNER_CONFIGS / "claude-code" / f"tier-{tier}" + overlay_src = tier_dir / "claude-overlay" + overlay_dst = fixture_dir / ".claude" + if overlay_dst.exists(): + shutil.rmtree(overlay_dst) + shutil.copytree(overlay_src, overlay_dst) + + transcript_path = run_dir / "transcripts" / f"{fixture_dir.name}_claude_t{tier}.txt" + transcript_path.parent.mkdir(parents=True, exist_ok=True) + + env = scrub_env() + cfg_dir = run_dir / "_claude_cfg" / f"{fixture_dir.name}_t{tier}" + cfg_dir.mkdir(parents=True, exist_ok=True) + env["CLAUDE_CONFIG_DIR"] = str(cfg_dir) + + proc = subprocess.run( + ["claude", "--print", "--dangerously-skip-permissions", prompt], + cwd=str(fixture_dir), env=env, capture_output=True, text=True, timeout=900, + ) + transcript_path.write_text( + f"# Cell: {fixture_dir.name} · claude · tier-{tier}\n" + f"# returncode: {proc.returncode}\n\n" + f"## stdout\n{proc.stdout}\n\n## stderr\n{proc.stderr}\n" + ) + return transcript_path, proc.returncode + + +def run_codex(fixture_dir: Path, tier: int, run_dir: Path, prompt: str) -> tuple[Path, int]: + tier_dir = RUNNER_CONFIGS / "codex" / f"tier-{tier}" + config = tier_dir / "config.toml" + sandbox = "read-only" if tier == 0 else "workspace-write" + + transcript_path = run_dir / "transcripts" / f"{fixture_dir.name}_codex_t{tier}.txt" + transcript_path.parent.mkdir(parents=True, exist_ok=True) + + strip_patterns = (r"/recce(/|$)", r"/dbt(/|$)", r"\.recce") if tier == 0 else (r"/dbt(/|$)",) + env = scrub_env() + env["PATH"] = scrub_path(strip_patterns) + # Tier-1 keeps single-env warehouse credentials from the parent shell; + # the operator is responsible for absence of base/prod creds (see codex/tier-1/README.md). + + proc = subprocess.run( + ["codex", "exec", + f"--sandbox={sandbox}", + "--ask-for-approval=never", + "--config", str(config), + prompt], + cwd=str(fixture_dir), env=env, capture_output=True, text=True, timeout=900, + ) + transcript_path.write_text( + f"# Cell: {fixture_dir.name} · codex · tier-{tier}\n" + f"# returncode: {proc.returncode}\n\n" + f"## stdout\n{proc.stdout}\n\n## stderr\n{proc.stderr}\n" + ) + return transcript_path, proc.returncode + + +def run_cell(cell: Cell, run_dir: Path) -> None: + fixture_dir = SOURCES_DIR / cell.fixture + if not fixture_dir.exists(): + cell.error = f"per-fixture worktree missing: {fixture_dir} (did you run build_fixtures.sh?)" + return + if not cli_available(cell.agent): + cell.error = f"{cell.agent} CLI not found on PATH; skipping" + return + try: + if cell.agent == "claude": + path, rc = run_claude(fixture_dir, cell.tier, run_dir, AGENT_PROMPT) + else: + path, rc = run_codex(fixture_dir, cell.tier, run_dir, AGENT_PROMPT) + cell.transcript_path = str(path) + cell.returncode = rc + except subprocess.TimeoutExpired: + cell.error = f"{cell.agent} timed out after 900s" + except FileNotFoundError as e: + cell.error = f"{cell.agent} setup failed: {e}" + + +def _shrink(text: str, limit: int = 30000) -> str: + if len(text) <= limit: + return text + half = limit // 2 + return text[:half] + "\n…[truncated]…\n" + text[-half:] + + +def judge_cell(cell: Cell, rubric: str, baseline_catch: str) -> dict: + if not cell.transcript_path: + return {"error": "no transcript"} + transcript = _shrink(Path(cell.transcript_path).read_text()) + user = JUDGE_USER_TEMPLATE.format( + rubric=_shrink(rubric, 8000), + fixture=cell.fixture, agent=cell.agent, tier=cell.tier, + transcript=transcript, baseline_catch=baseline_catch, + ) + proc = subprocess.run( + ["claude", "--print", "--dangerously-skip-permissions", + "--append-system-prompt", JUDGE_SYSTEM, user], + capture_output=True, text=True, timeout=300, + ) + if proc.returncode != 0: + return {"error": f"judge rc={proc.returncode}", "stderr": proc.stderr[:300]} + text = proc.stdout.strip() + start, end = text.find("{"), text.rfind("}") + if start < 0 or end < 0: + return {"error": "no JSON in judge output", "raw": text[:500]} + try: + return json.loads(text[start:end + 1]) + except json.JSONDecodeError as e: + return {"error": f"JSON parse: {e}", "raw": text[start:end + 1][:500]} + + +def parse_baseline_dir(baseline_dir: Path) -> dict[str, str]: + """Extract `catch / miss / partial` per fixture from tier-0-baseline.md files. + + Expects files at //tier-0-baseline.md following + the template at templates/tier-0-baseline.md. Returns {fixture: catch}. + """ + out: dict[str, str] = {} + if not baseline_dir or not baseline_dir.exists(): + return out + catch_re = re.compile(r"Catch\s*/\s*miss\s*/\s*partial:\s*`?(catch|miss|partial)`?", re.IGNORECASE) + for sub in baseline_dir.iterdir(): + baseline = sub / "tier-0-baseline.md" + if baseline.is_file(): + m = catch_re.search(baseline.read_text()) + if m: + out[sub.name] = m.group(1).lower() + return out + + +def axis_agreement(cells: list[Cell], axis: str) -> tuple[int, int]: + paired = [ + (c.verdict, c.verdict_2) for c in cells + if c.verdict and c.verdict_2 and "error" not in c.verdict and "error" not in c.verdict_2 + ] + if not paired: + return 0, 0 + agree = sum(1 for v1, v2 in paired if v1.get(axis) == v2.get(axis)) + return agree, len(paired) + + +def write_csv(cells: list[Cell], path: Path) -> None: + with path.open("w", newline="") as f: + w = csv.writer(f) + w.writerow([ + "fixture", "agent", "tier", + "catch", "evidence_tier", "delta", + "catch_2", "tier_2", "delta_2", + "returncode", "error", + ]) + for c in cells: + v = c.verdict or {} + v2 = c.verdict_2 or {} + w.writerow([ + c.fixture, c.agent, c.tier, + v.get("catch", ""), v.get("tier", ""), v.get("delta", ""), + v2.get("catch", ""), v2.get("tier", ""), v2.get("delta", ""), + "" if c.returncode is None else c.returncode, + c.error or "", + ]) + + +def write_summary(cells: list[Cell], path: Path, stability: bool, baseline: dict[str, str]) -> None: + lines = [ + "# Spike driver run — summary", + "", + f"- Run date: {dt.date.today().isoformat()}", + f"- Cells attempted: {len(cells)}", + f"- Cells with transcript: {sum(1 for c in cells if c.transcript_path)}", + f"- Cells judged: {sum(1 for c in cells if c.verdict and 'error' not in c.verdict)}", + f"- Cells errored: {sum(1 for c in cells if c.error)}", + "", + "## Cell matrix", + "", + "| Fixture | Agent | Tier | Catch | Evidence | Delta | Status |", + "|---|---|---|---|---|---|---|", + ] + for c in cells: + v = c.verdict or {} + status = c.error or ("judge_error" if v.get("error") else "ok") + lines.append( + f"| {c.fixture} | {c.agent} | {c.tier} | " + f"{v.get('catch','-')} | {v.get('tier','-')} | {v.get('delta','-')} | {status} |" + ) + + if stability: + lines += ["", "## Judge self-consistency (two passes on the same transcript)", ""] + for axis in ("catch", "tier", "delta"): + agree, total = axis_agreement(cells, axis) + pct = f"{agree / total:.0%}" if total else "n/a" + lines.append(f"- **{axis}**: {pct} ({agree}/{total} double-judged cells)") + lines += [ + "", + "**Stability bar:** ≥80% per axis. Below that, the judge can't replace human grading at N=6.", + ] + + if baseline: + lines += ["", "## Judge vs DRC-3585 manual baseline (catch axis)", ""] + compare = [ + (c, baseline[c.fixture]) + for c in cells if c.tier == 0 and c.fixture in baseline and c.verdict and "error" not in c.verdict + ] + if compare: + agree = sum(1 for c, b in compare if (c.verdict or {}).get("catch") == b) + lines.append(f"- Tier-0 cells with manual baseline available: {len(compare)}") + lines.append(f"- Judge–human catch agreement: {agree}/{len(compare)} ({agree / len(compare):.0%})") + else: + lines.append("- No Tier-0 cells overlap with baseline files; provide --baseline-dir pointing at fixtures/") + + path.write_text("\n".join(lines) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke", action="store_true", help="One fixture × all agents × all tiers") + parser.add_argument("--agents", default=",".join(AGENTS), help="comma-list of agents") + parser.add_argument("--tiers", default=",".join(str(t) for t in TIERS), help="comma-list of tiers") + parser.add_argument("--fixtures", default=",".join(DEFAULT_FIXTURES), help="comma-list of fixtures") + parser.add_argument("--judge-stability", action="store_true", help="Run judge twice per cell") + parser.add_argument("--baseline-dir", type=Path, help="DRC-3585 manual baseline dir; compares judge to human on catch axis") + parser.add_argument("--no-run", action="store_true", help="Skip agent runs; judge existing transcripts in --run-dir") + parser.add_argument("--run-dir", type=Path, help="Override run output dir (default runs//spike-driver/)") + args = parser.parse_args() + + agents = [a for a in args.agents.split(",") if a] + tiers = [int(t) for t in args.tiers.split(",") if t] + fixtures = list(args.fixtures.split(",")) + if args.smoke: + fixtures = fixtures[:1] + + run_dir = args.run_dir or (RUNS_DIR / dt.date.today().isoformat() / "spike-driver") + run_dir.mkdir(parents=True, exist_ok=True) + + rubric_path = EVAL_DIR / "RUBRIC.md" + rubric = rubric_path.read_text() if rubric_path.exists() else "(RUBRIC.md missing)" + baseline = parse_baseline_dir(args.baseline_dir) if args.baseline_dir else {} + if args.baseline_dir and not baseline: + print(f"[warn] --baseline-dir given but no baselines parsed from {args.baseline_dir}", file=sys.stderr) + + cells: list[Cell] = [ + Cell(fixture=f, agent=a, tier=t) + for f in fixtures for a in agents for t in tiers + ] + + if not args.no_run: + for cell in cells: + print(f"[run] {cell.fixture} · {cell.agent} · tier-{cell.tier}", file=sys.stderr) + run_cell(cell, run_dir) + if cell.error: + print(f" → {cell.error}", file=sys.stderr) + else: + # Re-judge mode: discover existing transcripts in run_dir + for cell in cells: + t = run_dir / "transcripts" / f"{cell.fixture}_{cell.agent}_t{cell.tier}.txt" + if t.exists(): + cell.transcript_path = str(t) + else: + cell.error = f"no existing transcript at {t}" + + for cell in cells: + if not cell.transcript_path: + continue + baseline_catch = baseline.get(cell.fixture, "unknown") + print(f"[judge] {cell.fixture} · {cell.agent} · tier-{cell.tier}", file=sys.stderr) + cell.verdict = judge_cell(cell, rubric, baseline_catch) + if args.judge_stability: + cell.verdict_2 = judge_cell(cell, rubric, baseline_catch) + + csv_path = run_dir / "verdicts.csv" + write_csv(cells, csv_path) + summary_path = run_dir / "summary.md" + write_summary(cells, summary_path, args.judge_stability, baseline) + cells_json = run_dir / "cells.json" + cells_json.write_text(json.dumps([asdict(c) for c in cells], indent=2)) + + print(f"\n[output] verdicts: {csv_path}", file=sys.stderr) + print(f"[output] summary: {summary_path}", file=sys.stderr) + print(f"[output] raw: {cells_json}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From c8952c45f90f633ec22913b8eaf62b0947aa3912 Mon Sep 17 00:00:00 2001 From: even-wei Date: Fri, 29 May 2026 17:43:10 +0800 Subject: [PATCH 2/4] fix(evals): stage frozen Tier-0 inputs into cwd before agent run The agent's cwd is .tmp/sources// (the per-fixture standalone repo), but the frozen Tier-0 inputs (diff.patch, artifacts/{manifest, compiled, catalog}*) live at fixtures//, outside cwd. Without staging: - Claude Code reaches them via absolute paths (Read tool isn't cwd-anchored), but the prompt doesn't tell the agent where to look. - Codex Tier-0 read-only sandbox blocks reads outside cwd entirely, so the agent literally cannot reach the artifacts. Driver now symlinks fixtures//{diff.patch, artifacts} into a _eval_inputs/ subdir under cwd before each cell. Prompt updated to point at _eval_inputs/. Works under both agent sandboxes. Verified locally: >>> stage_inputs(SOURCES_DIR / 'pr1-fix-clv', 'pr1-fix-clv') >>> sorted(p.name for p in (SOURCES_DIR / 'pr1-fix-clv' / '_eval_inputs').iterdir()) ['artifacts', 'diff.patch'] Signed-off-by: even-wei --- .../agent-blind-spots/spike-driver/driver.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/evals/agent-blind-spots/spike-driver/driver.py b/evals/agent-blind-spots/spike-driver/driver.py index c4c2067..f8f669e 100644 --- a/evals/agent-blind-spots/spike-driver/driver.py +++ b/evals/agent-blind-spots/spike-driver/driver.py @@ -56,13 +56,15 @@ TIERS = (0, 1) AGENT_PROMPT = ( - "Review this dbt PR. The inputs listed in the Tier-0 runtime contract " - "(diff.patch, manifest-before.json / manifest-after.json, compiled-before/ / " - "compiled-after/, catalog-before.json / catalog-after.json, and read access " - "to the dbt project at the head SHA) are available in the current working " - "directory. Decide catch / miss / partial per the rubric. Recommend " - "approve / request-changes / abstain. End your output with one line, " - "exactly:\n" + "Review this dbt PR. The current working directory is the dbt project at the " + "head SHA (models/, dbt_project.yml, etc.). The frozen Tier-0 inputs are " + "staged under `_eval_inputs/`:\n" + " - _eval_inputs/diff.patch — source-model diff base..head\n" + " - _eval_inputs/artifacts/manifest-{before,after}.json — dbt manifests pre/post\n" + " - _eval_inputs/artifacts/compiled-{before,after}/ — compiled SQL pre/post\n" + " - _eval_inputs/artifacts/catalog-{before,after}.json — schema-only (row stats are zero)\n" + "Decide catch / miss / partial per the rubric. Recommend " + "approve / request-changes / abstain. End your output with one line, exactly:\n" "VERDICT: · " ) @@ -129,6 +131,31 @@ def scrub_path(strip_patterns: tuple[str, ...]) -> str: return ":".join(p for p in os.environ.get("PATH", "").split(":") if not pat.search(p)) +def stage_inputs(fixture_dir: Path, fixture_id: str) -> None: + """Stage frozen Tier-0 inputs into the agent's cwd at `_eval_inputs/`. + + Symlinks `fixtures//{diff.patch, artifacts/}` into a `_eval_inputs/` + subdirectory of the per-fixture worktree, so an agent whose sandbox is + anchored on cwd (notably Codex Tier-0 read-only) can still reach the + frozen artifacts. Without this, only Claude Code (which doesn't anchor + Read on cwd) could see the inputs. + """ + src_root = EVAL_DIR / "fixtures" / fixture_id + if not src_root.exists(): + return + dst = fixture_dir / "_eval_inputs" + if dst.is_symlink() or dst.exists(): + if dst.is_symlink(): + dst.unlink() + else: + shutil.rmtree(dst) + dst.mkdir() + for name in ("diff.patch", "artifacts"): + src = src_root / name + if src.exists(): + (dst / name).symlink_to(src.resolve()) + + def run_claude(fixture_dir: Path, tier: int, run_dir: Path, prompt: str) -> tuple[Path, int]: tier_dir = RUNNER_CONFIGS / "claude-code" / f"tier-{tier}" overlay_src = tier_dir / "claude-overlay" @@ -195,6 +222,7 @@ def run_cell(cell: Cell, run_dir: Path) -> None: if not cli_available(cell.agent): cell.error = f"{cell.agent} CLI not found on PATH; skipping" return + stage_inputs(fixture_dir, cell.fixture) try: if cell.agent == "claude": path, rc = run_claude(fixture_dir, cell.tier, run_dir, AGENT_PROMPT) From 95a1bee923539275d2edf5b78492dfcab267df01 Mon Sep 17 00:00:00 2001 From: even-wei Date: Fri, 29 May 2026 17:50:37 +0800 Subject: [PATCH 3/4] fix(evals): inherit CLAUDE_CONFIG_DIR for auth in unattended runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ENFORCEMENT.md recipe step 3 sets CLAUDE_CONFIG_DIR to a fresh mktemp dir to neuter user-level ~/.claude/settings.json. That step also strips the Claude Code auth state, so a child `claude --print` invoked by the spike driver fails with "Not logged in" and no transcript ever lands. For unattended runs, the load-bearing enforcement is the project-level .claude/settings.json overlay (stamped per cell) plus the PreToolUse hook (deny-tier-N.py); a user-level `permissions.allow` cannot bypass an exit-2 hook. Skip the CLAUDE_CONFIG_DIR override by default. For paranoid mode (e.g. when running the eval on a machine with risky ~/.claude/ contents), set RECCE_EVAL_STRICT_CONFIG=1 to enable the override. The operator is responsible for preseeding auth under the per-cell _claude_cfg dir. Verified end-to-end smoke run after this fix: uv run driver.py --smoke --agents claude --tiers 0 [run] pr1-fix-clv · claude · tier-0 [judge] pr1-fix-clv · claude · tier-0 Agent VERDICT: catch · request-changes Judge verdict: {catch, tier-0, same} (delta=same expected with no T1 paired) Signed-off-by: even-wei --- .../agent-blind-spots/spike-driver/README.md | 19 +++++++++++++++++++ .../agent-blind-spots/spike-driver/driver.py | 15 ++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/evals/agent-blind-spots/spike-driver/README.md b/evals/agent-blind-spots/spike-driver/README.md index eea3060..4797d05 100644 --- a/evals/agent-blind-spots/spike-driver/README.md +++ b/evals/agent-blind-spots/spike-driver/README.md @@ -15,6 +15,25 @@ Not the durable harness. Output artifacts live under `runs//spike-driver/` - `codex` CLI on PATH (optional — codex cells skip gracefully if missing) - `uv` available (per repo convention) +## Auth note — `CLAUDE_CONFIG_DIR` + +`ENFORCEMENT.md` recipe step 3 sets `CLAUDE_CONFIG_DIR=$(mktemp -d)` to neuter +a stray `~/.claude/settings.json`. The driver **does not** apply that override +by default because it also strips the auth state — the child `claude --print` +fails with `Not logged in`. For unattended runs the load-bearing enforcement is +the project-level `.claude/settings.json` overlay (stamped per cell) plus the +`PreToolUse` hook; a user-level `permissions.allow` cannot bypass an exit-2 +hook regardless. + +To enable the strict override (paranoid mode), set: + +```bash +RECCE_EVAL_STRICT_CONFIG=1 uv run driver.py --smoke +``` + +You must preseed auth under the per-cell `_claude_cfg/_t/` dir +before each run; the driver does not provision auth. + ## Usage ```bash diff --git a/evals/agent-blind-spots/spike-driver/driver.py b/evals/agent-blind-spots/spike-driver/driver.py index f8f669e..fc37255 100644 --- a/evals/agent-blind-spots/spike-driver/driver.py +++ b/evals/agent-blind-spots/spike-driver/driver.py @@ -167,10 +167,19 @@ def run_claude(fixture_dir: Path, tier: int, run_dir: Path, prompt: str) -> tupl transcript_path = run_dir / "transcripts" / f"{fixture_dir.name}_claude_t{tier}.txt" transcript_path.parent.mkdir(parents=True, exist_ok=True) + # NOTE: ENFORCEMENT.md recipe step 3 (CLAUDE_CONFIG_DIR=$(mktemp -d)) is + # *intentionally not applied here* — neutering ~/.claude/ also strips the + # auth state, which breaks unattended runs. The load-bearing enforcement + # for Tier-0 is the project-level .claude/settings.json overlay (stamped + # above) + the PreToolUse hook (deny-tier-N.py); a stray user-level + # `permissions.allow` cannot bypass an exit-2 hook. For paranoid mode, + # set RECCE_EVAL_STRICT_CONFIG=1 to override CLAUDE_CONFIG_DIR; the cell + # will fail with "Not logged in" unless auth is preseeded under that dir. env = scrub_env() - cfg_dir = run_dir / "_claude_cfg" / f"{fixture_dir.name}_t{tier}" - cfg_dir.mkdir(parents=True, exist_ok=True) - env["CLAUDE_CONFIG_DIR"] = str(cfg_dir) + if os.environ.get("RECCE_EVAL_STRICT_CONFIG"): + cfg_dir = run_dir / "_claude_cfg" / f"{fixture_dir.name}_t{tier}" + cfg_dir.mkdir(parents=True, exist_ok=True) + env["CLAUDE_CONFIG_DIR"] = str(cfg_dir) proc = subprocess.run( ["claude", "--print", "--dangerously-skip-permissions", prompt], From 40282ee626f8489b18733df21e96dea18cb2369f Mon Sep 17 00:00:00 2001 From: even-wei Date: Fri, 29 May 2026 18:14:49 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(evals):=20address=20pr-cycle=20review?= =?UTF-8?q?=20of=20#37=20=E2=80=94=203=20ISSUEs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Drop dead VERDICT_TAIL_RE regex. Defined in driver.py:99 but never used; agent verdict parsing is fully handled by judge_cell() downstream. Removed alongside the regex line. 2. Add surgical .gitignore entries for spike-driver runtime outputs: evals/agent-blind-spots/runs/*/spike-driver/transcripts/ evals/agent-blind-spots/runs/*/spike-driver/_claude_cfg/ These are the genuinely-transient subdirs. verdicts.csv, summary.md, cells.json under runs//spike-driver/ are intentionally NOT ignored — they may be committed manually as run baselines for cross-iteration tracking. 3. stage_inputs() now raises FileNotFoundError when no Tier-0 inputs (neither diff.patch nor artifacts/) could be staged under the fixture's source dir. run_cell() catches and records into cell.error so the cell shows up as failed in the matrix instead of silently feeding the agent an empty _eval_inputs/ and letting it review with no inputs. Verified: python3 -m py_compile spike-driver/driver.py -> ok stage_inputs(.../pr1-fix-clv, 'pr1-fix-clv') -> happy path stage_inputs(/tmp/dne, 'does-not-exist') -> raises FileNotFoundError NOTE-level findings 4-7 from the review are accepted as-is per the spike framing (judge inherits parent env intentionally for auth; same-day run dirs are an operator concern; broken-symlink in overlay copy is rare and surfaced via run_cell's general except; no tests is consistent with the Karpathy-spike scope and the DRC-3587 follow-up which adds Inspect AI test harness). Signed-off-by: even-wei --- .gitignore | 7 ++++++ .../agent-blind-spots/spike-driver/driver.py | 22 +++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7f5fa70..0e59a40 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,13 @@ temp/ evals/agent-blind-spots/.tmp/ evals/agent-blind-spots/fixtures/*/artifacts/ +# DRC-3586 spike-driver runtime outputs — per-run transcripts and ephemeral +# Claude config dirs. The verdicts.csv / summary.md / cells.json under +# runs//spike-driver/ may be committed manually as run baselines, so +# the runs/ root is NOT ignored — only the transient subdirs are. +evals/agent-blind-spots/runs/*/spike-driver/transcripts/ +evals/agent-blind-spots/runs/*/spike-driver/_claude_cfg/ + # Coverage / editor logs .coverage .nvimlog diff --git a/evals/agent-blind-spots/spike-driver/driver.py b/evals/agent-blind-spots/spike-driver/driver.py index fc37255..80cba79 100644 --- a/evals/agent-blind-spots/spike-driver/driver.py +++ b/evals/agent-blind-spots/spike-driver/driver.py @@ -96,8 +96,6 @@ }} """ -VERDICT_TAIL_RE = re.compile(r"VERDICT:\s*(catch|miss|partial)\s*[·•|]?\s*(approve|request-changes|abstain)", re.IGNORECASE) - @dataclass class Cell: @@ -139,10 +137,14 @@ def stage_inputs(fixture_dir: Path, fixture_id: str) -> None: anchored on cwd (notably Codex Tier-0 read-only) can still reach the frozen artifacts. Without this, only Claude Code (which doesn't anchor Read on cwd) could see the inputs. + + Raises FileNotFoundError if neither `diff.patch` nor `artifacts/` exists + under `fixtures//`; otherwise the agent would see an empty + `_eval_inputs/` and review the PR with no Tier-0 inputs (silent miss). """ src_root = EVAL_DIR / "fixtures" / fixture_id if not src_root.exists(): - return + raise FileNotFoundError(f"fixture dir missing: {src_root}") dst = fixture_dir / "_eval_inputs" if dst.is_symlink() or dst.exists(): if dst.is_symlink(): @@ -150,10 +152,18 @@ def stage_inputs(fixture_dir: Path, fixture_id: str) -> None: else: shutil.rmtree(dst) dst.mkdir() + staged = 0 for name in ("diff.patch", "artifacts"): src = src_root / name if src.exists(): (dst / name).symlink_to(src.resolve()) + staged += 1 + if staged == 0: + dst.rmdir() + raise FileNotFoundError( + f"no Tier-0 inputs under {src_root} (need diff.patch and/or artifacts/); " + "did build_fixtures.sh complete for this fixture?" + ) def run_claude(fixture_dir: Path, tier: int, run_dir: Path, prompt: str) -> tuple[Path, int]: @@ -231,7 +241,11 @@ def run_cell(cell: Cell, run_dir: Path) -> None: if not cli_available(cell.agent): cell.error = f"{cell.agent} CLI not found on PATH; skipping" return - stage_inputs(fixture_dir, cell.fixture) + try: + stage_inputs(fixture_dir, cell.fixture) + except FileNotFoundError as e: + cell.error = f"stage_inputs failed: {e}" + return try: if cell.agent == "claude": path, rc = run_claude(fixture_dir, cell.tier, run_dir, AGENT_PROMPT)