From 5a7c483c35f322e6396828cc8b5b50e1b7ede441 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 09:21:15 -0400 Subject: [PATCH 1/8] WIP: version the diagnosis judge's template (tests pending) Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 46 +++ policybench/audit.py | 190 +++++++----- policybench/cli.py | 34 ++ policybench/judge_template.py | 147 +++++++++ scripts/run_audit_claude.sh | 17 +- scripts/run_audit_codex.sh | 21 +- tests/test_audit.py | 10 +- tests/test_finish_gpt61sol.py | 164 ++++++++-- tests/test_judge_provenance.py | 32 +- tests/test_judge_template.py | 545 +++++++++++++++++++++++++++++++++ tests/test_run_audit_claude.py | 22 ++ 11 files changed, 1113 insertions(+), 115 deletions(-) create mode 100644 policybench/judge_template.py create mode 100644 tests/test_judge_template.py diff --git a/docs/audit.md b/docs/audit.md index 9dfe007d..b1975cf9 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -47,6 +47,52 @@ side. It is told to default to `llm_error` and flag `reference_suspect` only with concrete evidence — the same conservatism as the deterministic inferrer, but with actual tax/benefit reasoning instead of keyword matching. +## Judge template versions + +Each prompt starts with a fixed header, the judge template, followed by the +case (`policybench/judge_template.py`). The template is versioned because a +release carries a seed verdict forward only when its case re-renders to the +exact bytes the judge read. + +| Version | Judged | Change | +|---|---|---| +| v1 | every verdict before versions were added (2026-10-09), through release dashboard-data-20261006 and the Claude Haiku 5.5 release's judges | none | +| v2 | new and re-opened cases from then on | drops v1's claim that the reference pipeline "has survived an adversarial review program" and that "the few real bugs found were fixed before this run" | + +The v1 claim is not so: release dashboard-data-20261006's exclusion record +stops scoring 28 outputs for engine defects, and the 2026-10-05 reference +adversary found four more (`reference_audit/2026-10-05-reference-adversary/` +on #200). + +- Both runners record the version a verdict was judged on in its + `verdict.meta.json` as `judge_template_version`. The Claude runner reads it + from the judged copy of `prompt.md`, the Codex runner from `prompt.md`. A + prompt that begins with no version's header, one `audit-prepare` did not + write, records `null`. +- A sidecar without the field, or a verdict without a sidecar, was judged on + v1. +- `audit-prepare` renders a case that already has a verdict with the version + its sidecar records. If that reproduces `prompt.md`, the verdict stands and + the prompt keeps its bytes. Otherwise the case changed since it was judged, + or its version is unknown: the verdict and its sidecar are dropped and the + case is re-opened. New and re-opened cases use the current version, or the + one `--template-version` names. +- The fold drivers (`scripts/finish_gpt61sol.py`, `scripts/finish_adds0928.py`) + copy the seed's prompts, verdicts and sidecars into their stage before they + call `prepare_audit`. So each seed case renders with its recorded version, + and a kept case's prompt keeps the seed's bytes. +- A tree may mix versions. `audit-collect` refuses a tree in which a verdict + records an unknown version, or one other than the version its `prompt.md` + begins with (`policybench.audit.template_version_problems`). Re-running + `audit-prepare` re-opens those cases. +- A version's text never changes once a verdict is judged on it; changing the + template means adding a version. `tests/test_judge_template.py` pins each + version's sha256. +- Every writer of `verdict.meta.json` must record the field. A writer that + does not, such as the Workflow and session judges of + `results/local/adds202609/judge_stages.py`, labels its verdicts v1; on a v2 + prompt, `audit-collect` refuses them and `audit-prepare` re-opens them. + ## Acting on suspect references Cases with `reference_suspect=true` are candidate PolicyEngine or data bugs. diff --git a/policybench/audit.py b/policybench/audit.py index 0c947b1d..a428c046 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -35,6 +35,14 @@ ) from policybench.case_annotations import _format_value, wrong_prediction_rows from policybench.full_run_export import load_case_reference_explanations +from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, + TEMPLATE_VERSION_FIELD, + recorded_template_version, + template_header, + template_version_of, +) from policybench.spec import metric_type_for_output # _format_value renders a missing/NaN prediction as this sentinel; such a miss @@ -275,67 +283,21 @@ def _scenario_questions( ], } -_PROMPT_HEADER = """\ -You are diagnosing why AI models missed a PolicyEngine reference value on a \ -US/UK tax-and-benefit estimation benchmark. The models answered from \ -parametric knowledge with no tools; PolicyEngine's microsimulation is the \ -reference. - -The reference value and its derivation are generated directly from the \ -engine's computation trace, and the reference pipeline has survived an \ -adversarial review program: every wrong-reference hypothesis raised by \ -earlier audits was adjudicated against primary sources, and the few real \ -bugs found were fixed before this run. Treat the reference and its \ -derivation as correct. Your job is NOT to re-litigate the reference — it is \ -to explain each model's mistake decisively. - -For every wrong model, write a `diagnosis`: 1-3 definitive sentences naming \ -the exact rule, eligibility pathway, deduction, threshold, or computation \ -step that model missed or misapplied, grounded in its own stated reasoning. \ -Be specific — "treated the 138% FPL MAGI limit as the only Medicaid pathway \ -and never applied the aged/disabled income test, which deducts the Medicare \ -Part B premium from countable income" — not generic ("got the income \ -calculation wrong"). If the model gave no usable reasoning, derive the \ -mistake from its answer: state what the correct derivation yields and what \ -shortcut the model's number is consistent with. - -Hedging is forbidden in `diagnosis` and `rationale`, and these phrasings \ -are mechanically rejected: "plausible"; "not enough evidence"; \ -"insufficient evidence/information"; "cannot/unable to \ -determine/verify/confirm/tell"; "difficult/hard to verify"; "may be/have"; \ -"might be/have"; "possibly"; "perhaps"; "unclear"; "the reference \ -is/appears/seems correct" or any other verdict on the reference. Write \ -definitively around them (e.g. "excess shelter costs are deductible", not \ -"may be deducted"). Doubt \ -about the reference belongs ONLY in reference_suspect + \ -reference_bug_hypothesis, and requires a concrete contradiction: the \ -derivation conflicts with a specific statute, regulation, or published \ -parameter you can name, or contradicts its own arithmetic. Absent that, set \ -reference_suspect=false and diagnose from the reference as ground truth. - -failure_source meanings: -- llm_error: the model reasoned or computed incorrectly (the usual case). -- prompt_ambiguity: the question is genuinely ambiguous; a careful expert \ -could read it more than one way. Name the two readings. -- reference_model_issue_fixed / reference_data_issue_fixed: the reference \ -value looks wrong (PolicyEngine logic / underlying data). Use with \ -reference_suspect=true and a concrete contradiction. -- parse_contract_failure: the model's answer was missing or unparseable, not a \ -substantive error. -- budget_exhausted_at_ceiling: the provider length-terminated every retry through \ -its maximum allowed completion budget. -- needs_review: genuinely cannot tell; name precisely what information is \ -missing. - -Output ONLY the JSON verdict matching the schema. Do not run any commands; all \ -information you need is below. -""" +# The judge's prompt is a versioned header (policybench.judge_template) and +# the case. A verdict is carried only on the exact bytes it was judged on, so +# a seed case renders with the version its sidecar records. + +def render_case_prompt( + case: AuditCase, template_version: int = CURRENT_TEMPLATE_VERSION +) -> str: + """Render the self-contained classification prompt for one case. -def render_case_prompt(case: AuditCase) -> str: - """Render the self-contained classification prompt for one case.""" + ``template_version`` picks the header (``JUDGE_TEMPLATE_HEADERS``); the + rest of the prompt does not depend on it. + """ lines = [ - _PROMPT_HEADER, + template_header(template_version), f"\nCOUNTRY: {case.country.upper()}", f"OUTPUT (variable): {case.variable} [{case.metric_type}]", f"POLICYENGINE REFERENCE VALUE: {case.reference_value}", @@ -368,6 +330,7 @@ def prepare_audit( country_dir: Path, audit_dir: Path, grounding_lookup: dict[tuple[str, str], str] | None = None, + template_version: int = CURRENT_TEMPLATE_VERSION, ) -> list[AuditCase]: """Write per-case prompts, the shared output schema, and a manifest. @@ -376,8 +339,17 @@ def prepare_audit( schema.json cases.jsonl cases//prompt.md - cases//verdict.json (written later by the runner) + cases//verdict.json (written later by the runner) + cases//verdict.meta.json (its provenance sidecar) + + A case that already has a verdict (a seed a release driver copied in, or + an earlier run's) is rendered with the template version its sidecar + records (absent: v1). If that reproduces its prompt.md, the verdict + stands and the prompt keeps its bytes. Otherwise the case changed since it + was judged: the verdict and its sidecar are dropped and the case is + re-opened. New and re-opened cases are rendered with ``template_version``. """ + template_header(template_version) cases = build_audit_cases(country_dir, grounding_lookup=grounding_lookup) audit_dir.mkdir(parents=True, exist_ok=True) (audit_dir / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA, indent=2)) @@ -405,24 +377,86 @@ def prepare_audit( case_dir.mkdir(exist_ok=True) prompt_path = case_dir / "prompt.md" verdict_path = case_dir / "verdict.json" - new_prompt = render_case_prompt(case) - # Content-aware resumability: if the case changed since it was last - # classified (e.g. a model was re-run and now answers differently), - # the prompt differs from the stored one — drop the stale verdict so - # the runner re-classifies it rather than reusing the old label. - if ( - verdict_path.exists() - and prompt_path.exists() - and prompt_path.read_text() != new_prompt - ): - verdict_path.unlink() - # The provenance sidecar describes that verdict; a re-judge by - # the other runner must not inherit it. - (case_dir / "verdict.meta.json").unlink(missing_ok=True) + new_prompt = None + if verdict_path.exists(): + # A judged case renders with the template its verdict was + # judged on, so an unchanged case keeps its prompt's bytes. + judged_on = recorded_template_version(_sidecar(case_dir)) + if judged_on is not None: + seeded = render_case_prompt(case, judged_on) + # A verdict without its prompt keeps it, as before + # templates were versioned. + if not prompt_path.exists() or prompt_path.read_text() == seeded: + new_prompt = seeded + if new_prompt is None: + # Content-aware resumability: the case changed since it was + # classified (e.g. a model was re-run and now answers + # differently), or its template is unknown. Drop the stale + # verdict so the runner re-classifies it rather than + # reusing the old label. The provenance sidecar describes + # that verdict; a re-judge by the other runner must not + # inherit it. + verdict_path.unlink() + (case_dir / "verdict.meta.json").unlink(missing_ok=True) + if new_prompt is None: + new_prompt = render_case_prompt(case, template_version) prompt_path.write_text(new_prompt) return cases +def _sidecar(case_dir: Path) -> dict | None: + """A case's ``verdict.meta.json``, or None when it has none. + + A sidecar that is not a JSON object records no template version. + """ + path = case_dir / "verdict.meta.json" + if not path.is_file(): + return None + try: + meta = json.loads(path.read_text()) + except ValueError: + meta = None + return meta if isinstance(meta, dict) else {TEMPLATE_VERSION_FIELD: None} + + +def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: + """Each judged case whose prompt and verdict disagree on the template. + + A tree may mix versions: a release's carried seeds keep the version they + were judged on, while its new and re-opened cases use the current one. + Every verdict must have its prompt.md, its sidecar must name a known + version (absent: v1), and prompt.md must begin with that version's + header. Returns ``(case_id, problem)`` for each case that fails, in case + order. + """ + problems: list[tuple[str, str]] = [] + cases_root = audit_dir / "cases" + if not cases_root.is_dir(): + return problems + for case_dir in sorted(cases_root.iterdir()): + if not (case_dir / "verdict.json").is_file(): + continue + meta = _sidecar(case_dir) + recorded = recorded_template_version(meta) + prompt_path = case_dir / "prompt.md" + if recorded is None: + problem = ( + f"its sidecar's {TEMPLATE_VERSION_FIELD} " + f"{meta.get(TEMPLATE_VERSION_FIELD)!r} names no template " + f"version ({sorted(JUDGE_TEMPLATE_HEADERS)})" + ) + elif not prompt_path.is_file(): + problem = "a verdict without prompt.md" + else: + actual = template_version_of(prompt_path.read_bytes()) + if actual == recorded: + continue + found = "no template version" if actual is None else f"v{actual}" + problem = f"prompt.md is {found}, but its verdict records v{recorded}" + problems.append((case_dir.name, problem)) + return problems + + # --- Hedge detection ----------------------------------------------------------- # Published diagnoses must be definitive. These patterns catch the judge @@ -516,14 +550,17 @@ def _row_failure_source(meta: dict, model: str, requested_source: str) -> str: def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame]: """Fold verdicts into the annotation schema. - Returns ``{"row": ..., "case": ..., "missing": ..., "hedged": ...}``. + Returns ``{"row": ..., "case": ..., "missing": ..., "hedged": ..., + "template": ...}``. ``row`` and ``case`` match the committed annotation CSV columns, extended with ``rationale`` and ``reference_suspect`` so the classifier's reasoning is preserved. ``missing`` lists cases whose verdict has not yet been produced (resumability). ``hedged`` lists case ids whose diagnosis or rationale hedges instead of diagnosing (see :data:`HEDGE_PATTERNS`) — delete those ``verdict.json`` files and re-run the classifier rather than - shipping them. + shipping them. ``template`` lists the judged cases whose prompt and + verdict disagree on the judge template version + (:func:`template_version_problems`). """ manifest = _load_manifest(audit_dir) cases_root = audit_dir / "cases" @@ -669,4 +706,7 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] "case": pd.DataFrame(case_records, columns=case_columns), "missing": pd.DataFrame({"case_id": missing}, columns=["case_id"]), "hedged": pd.DataFrame({"case_id": hedged}, columns=["case_id"]), + "template": pd.DataFrame( + template_version_problems(audit_dir), columns=["case_id", "problem"] + ), } diff --git a/policybench/cli.py b/policybench/cli.py index b2c8128f..f7990fb5 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -564,6 +564,13 @@ def main(): help="Optional CSV (scenario_id, variable, grounding) of authoritative " "engine facts rendered into matching case prompts", ) + audit_prepare_parser.add_argument( + "--template-version", + type=int, + default=None, + help="Judge template version for new and re-opened cases (default: the " + "current one; a judged case keeps the version its verdict records)", + ) audit_collect_parser = subparsers.add_parser( "audit-collect", @@ -1370,10 +1377,26 @@ def main(): for row in grounding_df.itertuples() if str(row.grounding).strip() } + from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, + ) + + template_version = ( + CURRENT_TEMPLATE_VERSION + if args.template_version is None + else args.template_version + ) + if template_version not in JUDGE_TEMPLATE_HEADERS: + raise SystemExit( + f"audit-prepare: no judge template version {template_version}; " + f"the versions are {sorted(JUDGE_TEMPLATE_HEADERS)}" + ) cases = prepare_audit( Path(args.country_dir), Path(args.audit_dir), grounding_lookup=grounding_lookup, + template_version=template_version, ) print( f"Prepared {len(cases)} audit cases under {args.audit_dir}. " @@ -1385,6 +1408,17 @@ def main(): country_dir = Path(args.country_dir) out = collect_audit(country_dir, Path(args.audit_dir)) + template = out["template"] + if not template.empty: + # A verdict judged on other words than its prompt.md's is not + # this case's verdict. + for case_id, problem in template.itertuples(index=False): + print(f"TEMPLATE: {case_id}: {problem}") + raise SystemExit( + f"{len(template)} verdicts disagree with their prompt.md on the " + "judge template version; re-run audit-prepare, which re-opens " + "them, and re-judge" + ) hedged = out["hedged"] if not hedged.empty and not args.allow_hedged: for case_id in hedged["case_id"]: diff --git a/policybench/judge_template.py b/policybench/judge_template.py new file mode 100644 index 00000000..41e6d0c6 --- /dev/null +++ b/policybench/judge_template.py @@ -0,0 +1,147 @@ +"""The diagnosis judge's prompt template, by version. + +Every audit prompt (:func:`policybench.audit.render_case_prompt`) is one of +these headers followed by the case. A version's text never changes once a +verdict has been judged on it: a release carries a seed verdict forward only +when its case re-renders to the exact bytes the judge read, so the renderer +must still produce every version a carried verdict was judged on. Changing +the template means adding a version. + +- **v1**, the only template before versions were added (2026-10-09), so + every verdict judged before then, through release dashboard-data-20261006 + and the Claude Haiku 5.5 release's judges. It tells the judge that the + reference pipeline "has survived an adversarial review program" and that + "the few real bugs found were fixed before this run". + That is not so: release dashboard-data-20261006's exclusion record stops + scoring 28 outputs for engine defects, and the 2026-10-05 reference + adversary found four more. +- **v2**, v1 without that claim. New cases and re-opened cases use it. + +Each verdict's provenance sidecar (``verdict.meta.json``) records the version +of the prompt it was judged on as ``judge_template_version``. A sidecar +without the field, or a verdict without a sidecar, predates the versions and +was judged on v1. + +Dependency-free, so the audit runners can read a prompt's version. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +# The sidecar field recording the version a verdict was judged on. +TEMPLATE_VERSION_FIELD = "judge_template_version" +# The version of a verdict whose sidecar does not record one. +UNRECORDED_TEMPLATE_VERSION = 1 +# The version new and re-opened cases are rendered with. +CURRENT_TEMPLATE_VERSION = 2 + +_V1 = """\ +You are diagnosing why AI models missed a PolicyEngine reference value on a \ +US/UK tax-and-benefit estimation benchmark. The models answered from \ +parametric knowledge with no tools; PolicyEngine's microsimulation is the \ +reference. + +The reference value and its derivation are generated directly from the \ +engine's computation trace, and the reference pipeline has survived an \ +adversarial review program: every wrong-reference hypothesis raised by \ +earlier audits was adjudicated against primary sources, and the few real \ +bugs found were fixed before this run. Treat the reference and its \ +derivation as correct. Your job is NOT to re-litigate the reference — it is \ +to explain each model's mistake decisively. + +For every wrong model, write a `diagnosis`: 1-3 definitive sentences naming \ +the exact rule, eligibility pathway, deduction, threshold, or computation \ +step that model missed or misapplied, grounded in its own stated reasoning. \ +Be specific — "treated the 138% FPL MAGI limit as the only Medicaid pathway \ +and never applied the aged/disabled income test, which deducts the Medicare \ +Part B premium from countable income" — not generic ("got the income \ +calculation wrong"). If the model gave no usable reasoning, derive the \ +mistake from its answer: state what the correct derivation yields and what \ +shortcut the model's number is consistent with. + +Hedging is forbidden in `diagnosis` and `rationale`, and these phrasings \ +are mechanically rejected: "plausible"; "not enough evidence"; \ +"insufficient evidence/information"; "cannot/unable to \ +determine/verify/confirm/tell"; "difficult/hard to verify"; "may be/have"; \ +"might be/have"; "possibly"; "perhaps"; "unclear"; "the reference \ +is/appears/seems correct" or any other verdict on the reference. Write \ +definitively around them (e.g. "excess shelter costs are deductible", not \ +"may be deducted"). Doubt \ +about the reference belongs ONLY in reference_suspect + \ +reference_bug_hypothesis, and requires a concrete contradiction: the \ +derivation conflicts with a specific statute, regulation, or published \ +parameter you can name, or contradicts its own arithmetic. Absent that, set \ +reference_suspect=false and diagnose from the reference as ground truth. + +failure_source meanings: +- llm_error: the model reasoned or computed incorrectly (the usual case). +- prompt_ambiguity: the question is genuinely ambiguous; a careful expert \ +could read it more than one way. Name the two readings. +- reference_model_issue_fixed / reference_data_issue_fixed: the reference \ +value looks wrong (PolicyEngine logic / underlying data). Use with \ +reference_suspect=true and a concrete contradiction. +- parse_contract_failure: the model's answer was missing or unparseable, not a \ +substantive error. +- budget_exhausted_at_ceiling: the provider length-terminated every retry through \ +its maximum allowed completion budget. +- needs_review: genuinely cannot tell; name precisely what information is \ +missing. + +Output ONLY the JSON verdict matching the schema. Do not run any commands; all \ +information you need is below. +""" + +# The claim v2 drops: the clause ends v1's second paragraph's first sentence. +V1_REVIEW_CLAIM = ( + ", and the reference pipeline has survived an adversarial review program: " + "every wrong-reference hypothesis raised by earlier audits was adjudicated " + "against primary sources, and the few real bugs found were fixed before " + "this run" +) +if _V1.count(V1_REVIEW_CLAIM) != 1: + raise AssertionError("v1 states the review claim exactly once") +_V2 = _V1.replace(V1_REVIEW_CLAIM, "", 1) + +JUDGE_TEMPLATE_HEADERS: dict[int, str] = {1: _V1, 2: _V2} + + +def template_header(version: int) -> str: + """The header a prompt of ``version`` begins with.""" + if type(version) is not int or version not in JUDGE_TEMPLATE_HEADERS: + raise ValueError( + f"no judge template version {version!r}; the versions are " + f"{sorted(JUDGE_TEMPLATE_HEADERS)}" + ) + return JUDGE_TEMPLATE_HEADERS[version] + + +def template_version_of(prompt: str | bytes) -> int | None: + """The version whose header begins ``prompt``, or None if none does. + + No header is a prefix of another, so at most one matches. + """ + if isinstance(prompt, bytes): + try: + prompt = prompt.decode("utf-8") + except UnicodeDecodeError: + return None + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + if prompt.startswith(header): + return version + return None + + +def recorded_template_version(meta: Mapping | None) -> int | None: + """The version a verdict's sidecar records it was judged on. + + No sidecar (None), or one without the field, means v1. A value that names + no version (null, a string, a boolean, an unknown number) gives None: the + verdict's template is unknown. + """ + if meta is None or TEMPLATE_VERSION_FIELD not in meta: + return UNRECORDED_TEMPLATE_VERSION + value = meta[TEMPLATE_VERSION_FIELD] + if type(value) is int and value in JUDGE_TEMPLATE_HEADERS: + return value + return None diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 490a7cc7..31f4190f 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -28,7 +28,10 @@ # an advisor model; and unless the judge's one accepted StructuredOutput # call (any other is one the schema refused) answered exactly the verdict; # - it reads a private copy of prompt.md, hashed before it runs; the sidecar -# records that hash. A verdict is rejected if prompt.md no longer has it +# records that hash and the judge template version the copy begins with +# (judge_template_version, policybench.judge_template; null for a prompt +# audit-prepare did not render). A verdict is rejected if prompt.md no +# longer has it # at extraction, or if the transcript's user events are not exactly one # text message equal to the judged prompt (plus Claude Code's own # StructuredOutput nudge, PROMPT_NUDGE) and the results of the judge's @@ -354,13 +357,18 @@ extract_verdict() { case_dir="$1"; out_tmp="$2"; meta_tmp="$3"; judged="$4"; judged_sha="$5" "$PYTHON" - "$case_dir" "$out_tmp" "$meta_tmp" "$MODEL" "$CLI_VERSION" \ "$CONFIG_DIR" "$AUTH" "$DECLARED" "$DISALLOWED" "$ATTACHMENTS" \ - "$EFFORT" "$judged" "$judged_sha" "$PROMPT_NUDGE" "$EVENT_TYPES" <<'PY' + "$EFFORT" "$judged" "$judged_sha" "$PROMPT_NUDGE" "$EVENT_TYPES" \ + "$SCRIPT_DIR" <<'PY' import datetime, glob, hashlib, json, re, shutil, sys from pathlib import Path (case_dir, out_path, meta_path, requested_model, cli_version, config_dir, auth, declared, disallowed, attachments, effort, judged_path, judged_sha, - nudge, event_types) = sys.argv[1:16] + nudge, event_types, script_dir) = sys.argv[1:17] +# The judge template versions, from this runner's own checkout. +sys.path.insert(0, str(Path(script_dir).resolve().parent)) +from policybench.judge_template import template_version_of + case = Path(case_dir) try: envelope = json.load(open(case / "claude.json")) @@ -597,6 +605,9 @@ meta = { "verdict_sha256": hashlib.sha256(verdict_bytes).hexdigest(), # The exact bytes the judge read on stdin, hashed before it ran. "prompt_sha256": judged_sha, + # The judge template those bytes begin with (policybench.judge_template), + # or null for a prompt audit-prepare did not render. + "judge_template_version": template_version_of(judged), "judge_model_requested": requested_model, "judge_model_reported": sorted((envelope.get("modelUsage") or {}).keys()), "judge_cli_version": cli_version, diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index dfb7d623..35716028 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -12,9 +12,10 @@ # # Judge provenance: beside each verdict.json the runner writes verdict.meta.json # (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex -# reports in its log header, the model requested, the UTC timestamp, and the +# reports in its log header, the model requested, the UTC timestamp, the # verdict's sha256, so a re-judged case can never keep the other runner's -# provenance. +# provenance, and the judge template version prompt.md begins with +# (policybench.judge_template; null for a prompt audit-prepare did not render). set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -85,7 +86,7 @@ classify_one() { -o "$tmp" \ - < "$prompt" > "$case_dir/codex.log" 2>&1 if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ - "$case_dir/verdict.meta.json.tmp"; then + "$case_dir/verdict.meta.json.tmp" "$prompt"; then mv -f "$tmp" "$out" mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" echo "[ok] $(basename "$case_dir")" @@ -97,12 +98,17 @@ classify_one() { # Provenance sidecar for a validated verdict: the model Codex reports in its # log header (`model: ...`), the model requested (or "default"), the UTC time, -# and the verdict's sha256 so the sidecar cannot outlive the verdict it describes. +# the verdict's sha256 so the sidecar cannot outlive the verdict it describes, +# and the judge template version of the prompt it was judged on. write_provenance() { - verdict_path="$1"; log_path="$2"; meta_out="$3" - "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" <<'PY' + verdict_path="$1"; log_path="$2"; meta_out="$3"; prompt_path="$4" + "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" \ + "$prompt_path" "$SCRIPT_DIR" <<'PY' import datetime, hashlib, json, re, sys -verdict_path, log_path, meta_out, requested = sys.argv[1:5] +from pathlib import Path +verdict_path, log_path, meta_out, requested, prompt_path, script_dir = sys.argv[1:7] +sys.path.insert(0, str(Path(script_dir).resolve().parent)) +from policybench.judge_template import template_version_of verdict = open(verdict_path, "rb").read() reported = [] try: @@ -117,6 +123,7 @@ meta = { "judge_model_requested": requested, "judge_model_reported": reported, "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "judge_template_version": template_version_of(open(prompt_path, "rb").read()), } json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True) PY diff --git a/tests/test_audit.py b/tests/test_audit.py index 3d4a3d76..525dcb91 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -518,10 +518,18 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): assert not verdict_path.exists() # stale verdict dropped assert not meta_path.exists() # and its provenance sidecar with it - # An unchanged case keeps its verdict. + # An unchanged case keeps its verdict, judged on the prompt's template + # (the re-opened case's prompt is the current version's). verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') + meta_path.write_text(json.dumps({"judge_template_version": 2})) prepare_audit(d, audit_dir) assert verdict_path.exists() + # A verdict whose sidecar records no version was judged on v1, so the + # current version's prompt is not the one it was judged on. + meta_path.write_text('{"judge_model_requested": "opus"}') + prepare_audit(d, audit_dir) + assert not verdict_path.exists() + assert not meta_path.exists() def test_collect_audit_coerces_parse_source_for_parsed_predictions(tmp_path: Path): diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index 6485d55e..cb8cf185 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -28,7 +28,15 @@ import finish_gpt61sol as driver # noqa: E402 -from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 +from policybench.audit import ( # noqa: E402 + AUDIT_OUTPUT_SCHEMA, + template_version_problems, +) +from policybench.judge_template import ( # noqa: E402 + CURRENT_TEMPLATE_VERSION, + V1_REVIEW_CLAIM, + template_version_of, +) NEW = "gpt-6.1-sol" SLUG = "gpt61sol" @@ -966,7 +974,9 @@ def seeded_stage(tmp_path, monkeypatch): lookup = {("s1", "snap"): "Gross test: pass."} seed = tmp_path / "seed" seed_board = _board(tmp_path / "seed-board", INCUMBENT_ROWS) - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + # Release 20260929's seed was judged on judge template v1, before versions + # existed, so its sidecars record none. + prepare_audit(seed_board / "us", seed, grounding_lookup=lookup, template_version=1) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], @@ -1015,16 +1025,32 @@ def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( assert (audit / "cases" / untouched / name).read_bytes() == ( seed / "cases" / untouched / name ).read_bytes() + # The kept case stays on the v1 template it was judged on; the re-opened + # one moves to the current template. + assert ( + template_version_of((audit / "cases" / untouched / "prompt.md").read_bytes()) + == 1 + ) + assert ( + template_version_of((audit / "cases" / joined / "prompt.md").read_bytes()) + == CURRENT_TEMPLATE_VERSION + ) assert json.loads((stage / "prompt-changes.json").read_text()) == { "added": [], "changed": [joined], "kept": [untouched], } wrong = ["m1", "m2", NEW] - write_verdict(audit / "cases" / joined, _verdict(wrong)) + write_verdict( + audit / "cases" / joined, + _verdict(wrong), + judge_template_version=CURRENT_TEMPLATE_VERSION, + ) seed_binding = driver.load_seed(stage) assert seed_binding == driver.seed_digest(seed) assert driver.validate_verdicts(audit, seed=seed_binding) == [] + # The stage mixes a v1 seed and a current-template verdict, and validates. + assert template_version_problems(audit) == [] # GPT-6.1 Sol misses s0 alone: s0 is re-opened, s1 carries over. @@ -1373,54 +1399,136 @@ def test_each_20260929_entry_names_its_seed_verdict_as_its_sidecar_records_it(): assert named_item(entry) == replaced_item(found, key in waved), entry -@pytest.mark.slow -def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): - """Release 20260929's 45-model board, as BASE_COMMIT holds it, renders - exactly the seed's cases and prompts. - - So check_prompt_changes may attribute every changed or new prompt in a - stage to GPT-6.1 Sol joining its case. #182's review excluded - scenario_023 head_medicaid_eligible and rewrote its adjudication and case - note; neither enters a prompt, and its prompt is unchanged. - """ - if not (SEED.is_dir() and GROUNDING.is_file()): - pytest.skip("release 20260929's audit or grounding is not on this machine") - from policybench.audit import prepare_audit - - bundle = tmp_path / "publish" / driver.RUN_NAME +def _committed_bundle(root: Path, blob) -> Path: + """The board a release committed, read through ``blob`` (a function from + a repository path to its bytes at that release's commit), as a bundle + prepare_audit can render: ``root/publish//us``.""" + bundle = root / "publish" / driver.RUN_NAME (bundle / "us").mkdir(parents=True) - # From git: after this release's freeze the working tree holds 20260930. for name in (*driver.REFERENCE_FILES, "predictions.csv.gz"): - (bundle / "us" / name).write_bytes( - driver.base_commit_blob((driver.SNAPSHOT / name).relative_to(driver.ROOT)) - ) + (bundle / "us" / name).write_bytes(blob(SNAPSHOT_PATH / name)) (bundle / "annotations").mkdir() for name in driver.ANNOTATION_FILES: (bundle / "annotations" / name).write_bytes( - driver.base_commit_blob( - (driver.ANNOTATIONS / name).relative_to(driver.ROOT) - ) + blob(Path("annotations") / driver.RUN_NAME / name) ) + return bundle / "us" + + +def _prepare_like_the_driver(country_dir: Path, audit: Path, **kwargs) -> None: + """prepare_audit with the grounding and the object strings the driver + renders with (resolve_base sets them).""" + from policybench.audit import prepare_audit + grounding = pd.read_csv(GROUNDING) lookup = { (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - audit = tmp_path / "audit" - # prepare renders with object strings, as resolve_base sets them. arrow = hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string") with ( pd.option_context("future.infer_string", False) if arrow else contextlib.nullcontext() ): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit(country_dir, audit, grounding_lookup=lookup, **kwargs) + + +@pytest.mark.slow +def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): + """Release 20260929's 45-model board, as BASE_COMMIT holds it, renders + exactly the seed's cases and prompts on judge template v1, the template + the seed was judged on. + + So check_prompt_changes may attribute every changed or new prompt in a + stage to GPT-6.1 Sol joining its case. #182's review excluded + scenario_023 head_medicaid_eligible and rewrote its adjudication and case + note; neither enters a prompt, and its prompt is unchanged. On the current + template every prompt is the seed's without v1's review claim. + """ + if not (SEED.is_dir() and GROUNDING.is_file()): + pytest.skip("release 20260929's audit or grounding is not on this machine") + # From git: after this release's freeze the working tree holds 20260930. + country_dir = _committed_bundle(tmp_path, driver.base_commit_blob) + audit = tmp_path / "audit" + _prepare_like_the_driver(country_dir, audit, template_version=1) assert (audit / "cases.jsonl").read_bytes() == (SEED / "cases.jsonl").read_bytes() rendered = driver.seed_prompt_digests(audit) assert rendered == driver.seed_prompt_digests(SEED) and len(rendered) == 674 case = "us__scenario_023__head_medicaid_eligible" meta = json.loads((SEED / "cases" / case / "verdict.meta.json").read_text()) assert rendered[case] == meta["prompt_sha256"] + current = tmp_path / "current" + _prepare_like_the_driver(country_dir, current) + for case_dir in sorted((SEED / "cases").iterdir()): + seed_prompt = (case_dir / "prompt.md").read_text() + assert (current / "cases" / case_dir.name / "prompt.md").read_text() == ( + seed_prompt.replace(V1_REVIEW_CLAIM, "", 1) + ), case_dir.name + + +# The audit trees releases froze (git-ignored, local only), each with the +# commit holding the board it was rendered from: release 20260929's (the +# GPT-6.1 Sol seed, BASE_COMMIT) and release 20260930's (RELEASE_COMMIT), +# which release dashboard-data-20261006 carried over unchanged. +FROZEN_AUDITS = { + "20260929": (SEED, "base"), + "20260930": ( + Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/gpt61sol-stage/" + "gpt61sol-v1/audit" + ), + "release", + ), +} + + +@pytest.mark.slow +@pytest.mark.parametrize("release", sorted(FROZEN_AUDITS)) +def test_every_frozen_verdict_rerenders_on_its_recorded_template(tmp_path, release): + """Local only: copied into a stage as prepare_cases copies a seed, every + verdict of a frozen audit tree re-renders on the template version its + sidecar records (absent: v1) to the exact prompt it was judged on, so + prepare_audit keeps every prompt, verdict and sidecar byte for byte under + the current default; and the tree validates.""" + from policybench.audit import template_version_problems + from policybench.judge_template import recorded_template_version + + tree, commit = FROZEN_AUDITS[release] + if not (tree.is_dir() and GROUNDING.is_file()): + pytest.skip(f"release {release}'s audit or grounding is not on this machine") + blob = driver.base_commit_blob if commit == "base" else release_blob + country_dir = _committed_bundle(tmp_path, blob) + stage = tmp_path / "stage" + judged = {} + for case_dir in sorted((tree / "cases").iterdir()): + if not (case_dir / "verdict.json").is_file(): + continue + target = stage / "cases" / case_dir.name + target.mkdir(parents=True) + judged[case_dir.name] = {} + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + if (case_dir / name).is_file(): + data = (case_dir / name).read_bytes() + (target / name).write_bytes(data) + judged[case_dir.name][name] = data + assert len(judged) == 674 + assert template_version_problems(stage) == [] + versions = { + case: recorded_template_version( + json.loads(files["verdict.meta.json"]) + if "verdict.meta.json" in files + else None + ) + for case, files in judged.items() + } + assert set(versions.values()) == {1} + _prepare_like_the_driver(country_dir, stage) + assert (stage / "cases.jsonl").read_bytes() == (tree / "cases.jsonl").read_bytes() + for case, files in judged.items(): + for name, data in files.items(): + assert (stage / "cases" / case / name).read_bytes() == data, (case, name) + assert template_version_problems(stage) == [] def test_check_prompt_changes_names_new_incumbent_only_cases(tmp_path): diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 00a6722d..5717c79c 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -21,7 +21,11 @@ verify_adjudications_keep_judge_verdicts, ) -from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 +from policybench.audit import ( # noqa: E402 + AUDIT_OUTPUT_SCHEMA, + template_version_problems, +) +from policybench.judge_template import template_header # noqa: E402 VERDICT = { "reference_suspect": False, @@ -289,6 +293,8 @@ def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): meta = json.loads((case_dir / "verdict.meta.json").read_text()) assert meta["judge_runner"] == "scripts/run_audit_codex.sh" assert meta["judge_model_reported"] == ["gpt-5.6-sol"] + # The fixture's prompt is no judge template's. + assert meta["judge_template_version"] is None assert ( meta["verdict_sha256"] == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() @@ -508,3 +514,27 @@ def test_adjudication_flag_matches_the_verdict_or_names_its_run(tmp_path: Path): [dict(entry, scenario_id="scenario_002", judge_reference_suspect=False)], cases, ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_both_runners_record_the_judge_template_version(tmp_path: Path): + """Each runner's sidecar records the template version of the prompt its + verdict was judged on, so prepare_audit can render the case on it again.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + for runner, version in (("run_audit_claude.sh", 2), ("run_audit_codex.sh", 1)): + (case_dir / "verdict.json").unlink(missing_ok=True) + (case_dir / "prompt.md").write_text( + template_header(version) + "\nCOUNTRY: US\nClassify this miss.\n" + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == f"scripts/{runner}" + assert meta["judge_template_version"] == version + assert template_version_problems(audit_dir) == [] diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py new file mode 100644 index 00000000..86816746 --- /dev/null +++ b/tests/test_judge_template.py @@ -0,0 +1,545 @@ +"""The diagnosis judge's template versions, and how a tree of verdicts keeps +the version each was judged on. + +Invariants: +- a version's header never changes (pinned by sha256), and no header is a + prefix of another, so a prompt's version is read off its bytes; +- the rest of a prompt does not depend on the version, and v2 is v1 without + the review claim; +- prepare_audit keeps a judged case's prompt, verdict and sidecar bytes when + the case re-renders to them with its recorded version (absent: v1), and + renders every other case with the requested version (default: current); +- after prepare_audit, a tree validates (template_version_problems is empty), + and a second prepare_audit changes nothing. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest import mock + +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policybench.audit import ( + AuditCase, + WrongModel, + build_audit_cases, + collect_audit, + prepare_audit, + render_case_prompt, + template_version_problems, +) +from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, + TEMPLATE_VERSION_FIELD, + UNRECORDED_TEMPLATE_VERSION, + V1_REVIEW_CLAIM, + recorded_template_version, + template_header, + template_version_of, +) + +ROOT = Path(__file__).resolve().parents[1] +# Properties run on a loaded machine too: no deadline, no slow-input check. +PROPERTY = settings( + deadline=None, suppress_health_check=[HealthCheck.too_slow], max_examples=50 +) +# Each version's header, by sha256 of its UTF-8 bytes. A carried verdict was +# judged on exactly these bytes: never edit a pin, add a version. +PINNED_HEADER_SHA256 = { + 1: "25b91bdcea41c37d4aa7d105d55d4b68cf39f34b153cc0fc877a2848317572e2", + 2: "7d8ff9b27d4b0341310267c11ea895f84d403a3219b6eddb8d0bf7dc4b1354ed", +} +# origin/main before template versions: policybench/audit.py's _PROMPT_HEADER +# there is the template every verdict until then was judged on. +UNVERSIONED_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" + + +# --- The versions -------------------------------------------------------------- + + +def test_every_version_is_pinned(): + assert set(JUDGE_TEMPLATE_HEADERS) == set(PINNED_HEADER_SHA256) + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + digest = hashlib.sha256(header.encode("utf-8")).hexdigest() + assert digest == PINNED_HEADER_SHA256[version], version + + +def test_v1_is_the_template_before_versions(): + """Differential: v1 is the header origin/main rendered before versions.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{UNVERSIONED_COMMIT}:policybench/audit.py"], + capture_output=True, + ) + if result.returncode: + pytest.fail( + f"cannot read policybench/audit.py at {UNVERSIONED_COMMIT[:12]}; fetch " + f"full history (git fetch --unshallow): {result.stderr.decode().strip()}" + ) + tree = ast.parse(result.stdout.decode("utf-8")) + (header,) = [ + node.value.value + for node in tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] == ["_PROMPT_HEADER"] + ] + assert JUDGE_TEMPLATE_HEADERS[1] == header + + +def test_the_current_version_is_the_newest_and_an_unrecorded_one_is_v1(): + assert CURRENT_TEMPLATE_VERSION == max(JUDGE_TEMPLATE_HEADERS) == 2 + assert UNRECORDED_TEMPLATE_VERSION == 1 + + +def test_v2_is_v1_without_the_review_claim(): + v1, v2 = JUDGE_TEMPLATE_HEADERS[1], JUDGE_TEMPLATE_HEADERS[2] + assert v1.count(V1_REVIEW_CLAIM) == 1 + assert v2 == v1.replace(V1_REVIEW_CLAIM, "", 1) + for phrase in ( + "has survived an adversarial review program", + "the few real bugs found were fixed before this run", + ): + assert phrase in v1 and phrase not in v2 + assert ( + "generated directly from the engine's computation trace. Treat the " + "reference and its derivation as correct." in " ".join(v2.split()) + ) + + +def test_no_header_is_a_prefix_of_another(): + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + for other, other_header in JUDGE_TEMPLATE_HEADERS.items(): + if version != other: + assert not other_header.startswith(header), (version, other) + + +def test_an_unknown_version_cannot_be_rendered(): + for version in (0, 3, True, "1", None, 1.0): + with pytest.raises(ValueError, match="no judge template version"): + template_header(version) + + +@PROPERTY +@given( + st.one_of( + st.none(), + st.booleans(), + st.integers(-5, 10), + st.floats(allow_nan=False), + st.text(max_size=3), + st.lists(st.integers(), max_size=2), + ) +) +def test_recorded_version_is_the_field_when_it_names_one(value): + expected = value if type(value) is int and value in JUDGE_TEMPLATE_HEADERS else None + assert recorded_template_version({TEMPLATE_VERSION_FIELD: value}) == expected + assert recorded_template_version({"judge_runner": "x"}) == 1 + assert recorded_template_version(None) == 1 + + +# --- Rendering ------------------------------------------------------------------- + +TEXT = st.text( + alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\r"), + max_size=60, +) + + +@st.composite +def audit_cases(draw) -> AuditCase: + models = draw( + st.lists( + st.from_regex(r"[a-z][a-z0-9.-]{0,12}", fullmatch=True), + min_size=1, + max_size=4, + unique=True, + ) + ) + return AuditCase( + case_id=draw(st.from_regex(r"us__s[0-9]{1,3}__[a-z_]{1,12}", fullmatch=True)), + country=draw(st.sampled_from(["us", "uk"])), + scenario_id=draw(st.from_regex(r"s[0-9]{1,3}", fullmatch=True)), + variable=draw(st.from_regex(r"[a-z_]{1,20}", fullmatch=True)), + metric_type=draw(st.sampled_from(["amount", "binary"])), + reference_value=draw(st.sampled_from(["$0", "$1,234", "Yes", "No"])), + reference_derivation=draw(TEXT), + question=draw(TEXT), + wrong_models=tuple( + WrongModel( + model=model, + prediction=draw(st.sampled_from(["$250", "$0", "missing", "No"])), + explanation=draw(TEXT), + ) + for model in models + ), + grounding=draw(TEXT), + ) + + +@PROPERTY +@given(audit_cases()) +def test_a_prompts_version_is_read_off_its_bytes(case): + for version in JUDGE_TEMPLATE_HEADERS: + prompt = render_case_prompt(case, version) + assert prompt.startswith(template_header(version)) + assert template_version_of(prompt) == version + assert template_version_of(prompt.encode("utf-8")) == version + + +@PROPERTY +@given(audit_cases()) +def test_only_the_header_depends_on_the_version(case): + bodies = { + render_case_prompt(case, version)[len(header) :] + for version, header in JUDGE_TEMPLATE_HEADERS.items() + } + assert len(bodies) == 1 + v1, v2 = render_case_prompt(case, 1), render_case_prompt(case, 2) + assert v2 == v1.replace(V1_REVIEW_CLAIM, "", 1) + assert render_case_prompt(case) == render_case_prompt( + case, CURRENT_TEMPLATE_VERSION + ) + + +@PROPERTY +@given(st.binary(max_size=80)) +def test_a_prompt_without_a_header_has_no_version(data): + if not any(data.startswith(h.encode()) for h in JUDGE_TEMPLATE_HEADERS.values()): + assert template_version_of(data) is None + assert template_version_of(b"\xff\xfe" + data) is None + + +# --- prepare_audit --------------------------------------------------------------- + + +def _board(directory: Path, answers: dict[str, float]) -> Path: + """A US run in which model m1 misses each scenario's $0 SNAP reference + with the given answer.""" + directory.mkdir(parents=True, exist_ok=True) + pd.DataFrame( + [{"scenario_id": s, "variable": "snap", "value": 0.0} for s in answers] + ).to_csv(directory / "reference_outputs.csv", index=False) + pd.DataFrame( + [ + { + "model": "m1", + "scenario_id": s, + "variable": "snap", + "prediction": answer, + "explanation": f"Estimated {answer}.", + "error": None, + } + for s, answer in answers.items() + ] + ).to_csv(directory / "predictions.csv", index=False) + return directory + + +VERDICT = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": "Used the gross income test alone.", + "models": [ + { + "model": "m1", + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": "Applied the 2025 threshold.", + } + ], +} +# How a seed case's verdict records its template: the sidecar it gets. +SEED_STATES = { + "unjudged": None, + "no sidecar": None, + "v1, field absent": {}, + "v1": {TEMPLATE_VERSION_FIELD: 1}, + "v2": {TEMPLATE_VERSION_FIELD: 2}, + "unknown (null)": {TEMPLATE_VERSION_FIELD: None}, + "unknown (99)": {TEMPLATE_VERSION_FIELD: 99}, + "unknown (true)": {TEMPLATE_VERSION_FIELD: True}, +} +# The version a state's prompt was rendered with: an unjudged case's is the +# current one, as prepare_audit writes it; an unknown one's is v1's bytes. +SEED_PROMPT_VERSION = { + "unjudged": CURRENT_TEMPLATE_VERSION, + "no sidecar": 1, + "v1, field absent": 1, + "v1": 1, + "v2": 2, + "unknown (null)": 1, + "unknown (99)": 1, + "unknown (true)": 1, +} + + +def _judge(case_dir: Path, sidecar: dict | None) -> None: + """Write a verdict and, unless ``sidecar`` is None, its sidecar.""" + path = case_dir / "verdict.json" + path.write_text(json.dumps(VERDICT, indent=2, sort_keys=True)) + if sidecar is not None: + meta = { + "judge_runner": "scripts/run_audit_claude.sh", + "verdict_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + **sidecar, + } + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + + +def _seed(audit: Path, board: Path, states: dict[str, str]) -> dict[str, AuditCase]: + """Prepare ``board`` into ``audit``, then give each scenario's case the + prompt and verdict its state names. Returns the cases by scenario.""" + prepare_audit(board, audit) + cases = {case.scenario_id: case for case in build_audit_cases(board)} + for scenario, state in states.items(): + case_dir = audit / "cases" / cases[scenario].case_id + (case_dir / "prompt.md").write_text( + render_case_prompt(cases[scenario], SEED_PROMPT_VERSION[state]) + ) + if state != "unjudged": + _judge(case_dir, SEED_STATES[state]) + return cases + + +def _tree(audit: Path) -> dict[str, bytes]: + return { + str(path.relative_to(audit)): path.read_bytes() + for path in sorted(audit.rglob("*")) + if path.is_file() + } + + +@settings(max_examples=60, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given( + st.dictionaries( + st.sampled_from(["s0", "s1", "s2", "s3"]), + st.tuples(st.sampled_from(sorted(SEED_STATES)), st.booleans()), + min_size=1, + ), + st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), +) +def test_prepare_keeps_each_judged_case_on_its_own_template(plan, version): + """Property over seed trees: a judged case whose case is unchanged keeps + its prompt, verdict and sidecar bytes when its recorded version is known; + every other case is (re-)opened on ``version``. The prepared tree + validates, and preparing again changes nothing.""" + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + board = _board(root / "us", {s: 250.0 for s in plan}) + audit = root / "audit" + states = {scenario: state for scenario, (state, _) in plan.items()} + before_cases = _seed(audit, board, states) + before = _tree(audit) + # Re-run m1 on the scenarios the plan changes: their prompts change. + changed = {s for s, (_, change) in plan.items() if change} + _board(board, {s: 999.0 if s in changed else 250.0 for s in plan}) + prepare_audit(board, audit, template_version=version) + cases = {case.scenario_id: case for case in build_audit_cases(board)} + after = _tree(audit) + for scenario, state in states.items(): + case_dir = f"cases/{cases[scenario].case_id}" + assert cases[scenario].case_id == before_cases[scenario].case_id + recorded = ( + recorded_template_version(SEED_STATES[state]) + if state != "unjudged" + else None + ) + carried = recorded is not None and scenario not in changed + if carried: + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + key = f"{case_dir}/{name}" + assert after.get(key) == before.get(key), (state, name) + assert template_version_of(after[f"{case_dir}/prompt.md"]) == recorded + else: + assert f"{case_dir}/verdict.json" not in after, state + assert f"{case_dir}/verdict.meta.json" not in after, state + assert after[f"{case_dir}/prompt.md"] == render_case_prompt( + cases[scenario], version + ).encode("utf-8") + assert template_version_problems(audit) == [] + prepare_audit(board, audit, template_version=version) + assert _tree(audit) == after + + +def test_new_cases_get_the_current_version(tmp_path): + board = _board(tmp_path / "us", {"s0": 250.0, "s1": 100.0}) + audit = tmp_path / "audit" + cases = prepare_audit(board, audit) + for case in cases: + prompt = (audit / "cases" / case.case_id / "prompt.md").read_bytes() + assert template_version_of(prompt) == CURRENT_TEMPLATE_VERSION == 2 + assert b"fixed before this run" not in prompt + assert b"survived an adversarial review program" not in prompt + + +def test_a_requested_version_renders_new_cases_and_an_unknown_one_is_refused( + tmp_path, +): + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + prompt = (audit / "cases" / case.case_id / "prompt.md").read_text() + assert prompt == render_case_prompt(case, 1) + assert "fixed before this run" in prompt + with pytest.raises(ValueError, match="no judge template version 3"): + prepare_audit(board, tmp_path / "other", template_version=3) + assert not (tmp_path / "other").exists() + + +def test_a_v1_seed_carries_over_and_a_reopened_case_moves_to_v2(tmp_path): + """The fold drivers' flow: a seed judged on v1 (its sidecar predates the + field) is copied into a stage; the case a new model joins is re-opened on + v2, and the other keeps the seed's bytes.""" + seed_board = _board(tmp_path / "seed" / "us", {"s0": 250.0, "s1": 100.0}) + seed = tmp_path / "seed" / "audit" + prepare_audit(seed_board, seed, template_version=1) + for case_dir in (seed / "cases").iterdir(): + _judge(case_dir, {}) + stage = tmp_path / "stage" / "audit" + stage.mkdir(parents=True) + for case_dir in (seed / "cases").iterdir(): + target = stage / "cases" / case_dir.name + target.mkdir(parents=True) + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + (target / name).write_bytes((case_dir / name).read_bytes()) + board = _board(tmp_path / "stage" / "us", {"s0": 250.0, "s1": 100.0}) + # A new model misses s0 and answers s1's $0 reference exactly. + predictions = pd.read_csv(board / "predictions.csv") + new = predictions.assign(model="new") + new.loc[new.scenario_id == "s1", "prediction"] = 0.0 + pd.concat([predictions, new]).to_csv(board / "predictions.csv", index=False) + cases = {case.scenario_id: case for case in prepare_audit(board, stage)} + kept, reopened = (stage / "cases" / cases[s].case_id for s in ("s1", "s0")) + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + assert (kept / name).read_bytes() == ( + seed / "cases" / kept.name / name + ).read_bytes() + assert template_version_of((kept / "prompt.md").read_bytes()) == 1 + assert not (reopened / "verdict.json").exists() + assert (reopened / "prompt.md").read_text() == render_case_prompt(cases["s0"], 2) + assert template_version_problems(stage) == [] + + +# --- A mixed tree ---------------------------------------------------------------- + + +def _mixed_tree(tmp_path: Path) -> tuple[Path, Path, dict[str, AuditCase]]: + board = _board(tmp_path / "us", {f"s{i}": 250.0 + i for i in range(5)}) + audit = tmp_path / "audit" + cases = _seed( + audit, + board, + { + "s0": "no sidecar", + "s1": "v1, field absent", + "s2": "v1", + "s3": "v2", + "s4": "unjudged", + }, + ) + return board, audit, cases + + +def test_a_mixed_tree_validates_and_collects(tmp_path): + board, audit, cases = _mixed_tree(tmp_path) + assert template_version_problems(audit) == [] + out = collect_audit(board, audit) + assert out["template"].empty + assert list(out["template"].columns) == ["case_id", "problem"] + assert sorted(out["missing"]["case_id"]) == [cases["s4"].case_id] + assert len(out["case"]) == 4 + # Preparing it again keeps every verdict: each case renders to its bytes. + before = _tree(audit) + prepare_audit(board, audit) + assert _tree(audit) == before + _collect_cli(board, audit, tmp_path / "out") + assert (tmp_path / "out" / "us_audit_case_annotations.csv").is_file() + + +def test_a_verdict_whose_template_disagrees_with_its_prompt_is_flagged(tmp_path): + board, audit, cases = _mixed_tree(tmp_path) + case_dir = {s: audit / "cases" / case.case_id for s, case in cases.items()} + + def meta(scenario: str, **fields) -> None: + path = case_dir[scenario] / "verdict.meta.json" + path.write_text(json.dumps({**json.loads(path.read_text()), **fields})) + + meta("s1", judge_template_version=2) # records v2 on a v1 prompt + sidecar = case_dir["s3"] / "verdict.meta.json" + stripped = json.loads(sidecar.read_text()) + del stripped[TEMPLATE_VERSION_FIELD] + sidecar.write_text(json.dumps(stripped)) # v2 prompt, field absent (v1) + meta("s2", judge_template_version="2") # names no version + (case_dir["s0"] / "prompt.md").unlink() # a verdict without its prompt + (case_dir["s4"] / "verdict.json").write_text(json.dumps(VERDICT)) + (case_dir["s4"] / "prompt.md").write_text("Classify this miss.\n") + problems = dict(template_version_problems(audit)) + assert problems == { + cases["s0"].case_id: "a verdict without prompt.md", + cases["s1"].case_id: "prompt.md is v1, but its verdict records v2", + cases["s2"].case_id: ( + "its sidecar's judge_template_version '2' names no template " + "version ([1, 2])" + ), + cases["s3"].case_id: "prompt.md is v2, but its verdict records v1", + cases["s4"].case_id: "prompt.md is no template version, but its verdict " + "records v1", + } + out = collect_audit(board, audit) + assert dict(out["template"].itertuples(index=False)) == problems + with pytest.raises(SystemExit, match="5 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + # Preparing again re-opens every flagged case except the verdict that + # lost its prompt, which keeps its verdict and gets its v1 prompt back. + prepare_audit(board, audit) + assert template_version_problems(audit) == [] + assert (case_dir["s0"] / "verdict.json").is_file() + assert (case_dir["s0"] / "prompt.md").read_text() == render_case_prompt( + cases["s0"], 1 + ) + for scenario in ("s1", "s2", "s3", "s4"): + assert not (case_dir[scenario] / "verdict.json").exists(), scenario + assert (case_dir[scenario] / "prompt.md").read_text() == render_case_prompt( + cases[scenario], 2 + ) + + +def _collect_cli(board: Path, audit: Path, out: Path) -> None: + from policybench import cli + + argv = ["policybench", "audit-collect", "--country-dir", str(board)] + argv += ["--audit-dir", str(audit), "--output-dir", str(out)] + with mock.patch.object(sys, "argv", argv): + cli.main() + + +def test_audit_prepare_takes_a_template_version(tmp_path, capsys): + from policybench import cli + + board = _board(tmp_path / "us", {"s0": 250.0}) + for flag, version in ((["--template-version", "1"], 1), ([], 2)): + audit = tmp_path / f"audit-{version}" + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(audit), *flag] + with mock.patch.object(sys, "argv", argv): + cli.main() + (prompt,) = (audit / "cases").glob("*/prompt.md") + assert template_version_of(prompt.read_bytes()) == version + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(tmp_path / "audit-9"), "--template-version", "9"] + with mock.patch.object(sys, "argv", argv): + with pytest.raises(SystemExit, match="no judge template version 9"): + cli.main() diff --git a/tests/test_run_audit_claude.py b/tests/test_run_audit_claude.py index c3c42839..f944801f 100644 --- a/tests/test_run_audit_claude.py +++ b/tests/test_run_audit_claude.py @@ -24,6 +24,7 @@ from hypothesis import strategies as st from policybench.audit import AUDIT_OUTPUT_SCHEMA +from policybench.judge_template import JUDGE_TEMPLATE_HEADERS, template_header ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) @@ -370,6 +371,26 @@ def test_the_sidecar_binds_the_prompt_and_records_the_login(lane): assert f"Classify {name}." in transcript +@pytest.mark.parametrize("version", sorted(JUDGE_TEMPLATE_HEADERS)) +def test_the_sidecar_records_the_judge_template_version(lane, version): + """The sidecar records the template version the judged prompt begins with, + and null for a prompt audit-prepare did not render.""" + audit, _, _, run = lane + rendered, fixture = CASES + (audit / "cases" / rendered / "prompt.md").write_text( + template_header(version) + f"\nCOUNTRY: US\nClassify {rendered}.\n" + ) + result, _ = run() + assert result.returncode == 0, result.stderr + result.stdout + recorded = { + name: json.loads((audit / "cases" / name / "verdict.meta.json").read_text())[ + "judge_template_version" + ] + for name in CASES + } + assert recorded == {rendered: version, fixture: None} + + def test_a_home_lanes_own_account_is_recorded(lane): audit, _, _, run = lane home_login = {**DESKTOP, "email": "lane@x.org"} @@ -1239,6 +1260,7 @@ def _judge_both(work: Path, events: list, prompt: bytes, monkeypatch): argv += [str(config), "{}", "", _runner_value("DISALLOWED")] argv += [_runner_value("ATTACHMENTS"), "xhigh", str(work / "judged")] argv += [sha(prompt), _runner_value("PROMPT_NUDGE"), _runner_value("EVENT_TYPES")] + argv += [str(RUNNER.parent)] monkeypatch.setattr(sys, "argv", ["extract_verdict", *argv]) try: with contextlib.redirect_stderr(io.StringIO()): From b11a959b5de840307d610daa4014fc145bfa7ea5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 05:59:11 -0400 Subject: [PATCH 2/8] Name the judge template version at every renderer and test the invariants render_case_prompt, prepare_audit and audit-prepare now require the template version (keyword-only, no default; --template-version is required). The three past release drivers name JUDGE_TEMPLATE_VERSION = 1, the version their releases were judged on, so re-running one reproduces its prompts; the Haiku 5.5 driver's tests seed on v1. Tests state and check the invariants: every committed prompt sha256 (docs/gpt61sol, docs/haiku55) re-renders from its release's committed board on its recorded version (local, needs the grounding); v2 is v1 with one deletion, the review clause; selection is explicit (signatures, an AST scan of every call, and a Hypothesis property that a prompt on another version is re-opened, never adopted). Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 59 ++++- policybench/audit.py | 31 +-- policybench/cli.py | 27 +- policybench/judge_template.py | 20 +- scripts/finish_adds0928.py | 12 +- scripts/finish_gpt61sol.py | 12 +- scripts/finish_haiku55.py | 28 ++- tests/test_audit.py | 65 +++-- tests/test_finish_gpt61sol.py | 45 ++-- tests/test_finish_haiku55.py | 15 +- tests/test_judge_template.py | 459 +++++++++++++++++++++++++++++++--- 11 files changed, 647 insertions(+), 126 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index b1975cf9..6c3c2f85 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -18,7 +18,8 @@ Python (`policybench/audit.py`). # 1. Assemble one prompt per wrong case (+ shared output schema + manifest). uv run python -m policybench.cli audit-prepare \ --country-dir results//us \ - --audit-dir results//us/audit + --audit-dir results//us/audit \ + --template-version 2 # 2. Classify in bulk via Codex (ChatGPT plan). Resumable + parallel. AUDIT_PARALLEL=4 AUDIT_REASONING_EFFORT=low \ @@ -56,31 +57,42 @@ exact bytes the judge read. | Version | Judged | Change | |---|---|---| -| v1 | every verdict before versions were added (2026-10-09), through release dashboard-data-20261006 and the Claude Haiku 5.5 release's judges | none | -| v2 | new and re-opened cases from then on | drops v1's claim that the reference pipeline "has survived an adversarial review program" and that "the few real bugs found were fixed before this run" | +| v1 | every verdict through release dashboard-data-20261010 (Claude Haiku 5.5), the last release judged before versions existed | none | +| v2 | the version a new audit names | drops v1's claim that the reference pipeline "has survived an adversarial review program" and that "the few real bugs found were fixed before this run" | The v1 claim is not so: release dashboard-data-20261006's exclusion record stops scoring 28 outputs for engine defects, and the 2026-10-05 reference -adversary found four more (`reference_audit/2026-10-05-reference-adversary/` -on #200). - +adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/`, +#200). + +- Version selection is explicit. `render_case_prompt` and `prepare_audit` take + `template_version` as a required keyword, and `audit-prepare` requires + `--template-version`; nothing defaults to `CURRENT_TEMPLATE_VERSION`. A + judged case renders on the version its sidecar records. Nothing reads a + version off a prompt to choose how to render it: `template_version_of` only + records what a judge read and checks a tree. - Both runners record the version a verdict was judged on in its `verdict.meta.json` as `judge_template_version`. The Claude runner reads it from the judged copy of `prompt.md`, the Codex runner from `prompt.md`. A prompt that begins with no version's header, one `audit-prepare` did not write, records `null`. - A sidecar without the field, or a verdict without a sidecar, was judged on - v1. + v1 (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars + written before versions existed, not a guess from the prompt. - `audit-prepare` renders a case that already has a verdict with the version its sidecar records. If that reproduces `prompt.md`, the verdict stands and the prompt keeps its bytes. Otherwise the case changed since it was judged, or its version is unknown: the verdict and its sidecar are dropped and the - case is re-opened. New and re-opened cases use the current version, or the - one `--template-version` names. -- The fold drivers (`scripts/finish_gpt61sol.py`, `scripts/finish_adds0928.py`) - copy the seed's prompts, verdicts and sidecars into their stage before they - call `prepare_audit`. So each seed case renders with its recorded version, - and a kept case's prompt keeps the seed's bytes. + case is re-opened. A `prompt.md` on another version is re-opened, not + adopted. New and re-opened cases use the version the caller names. +- Each release driver names the version its new and re-opened cases render + on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before + versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, + `scripts/finish_haiku55.py`) name v1, so re-running one reproduces its + release's prompts. A new release's driver names v2. Each driver copies the + seed's prompts, verdicts and sidecars into its stage before it calls + `prepare_audit`, so each seed case renders on its recorded version and a + kept case's prompt keeps the seed's bytes. - A tree may mix versions. `audit-collect` refuses a tree in which a verdict records an unknown version, or one other than the version its `prompt.md` begins with (`policybench.audit.template_version_problems`). Re-running @@ -93,6 +105,27 @@ on #200). `results/local/adds202609/judge_stages.py`, labels its verdicts v1; on a v2 prompt, `audit-collect` refuses them and `audit-prepare` re-opens them. +### Invariants + +`tests/test_judge_template.py` states and tests these: + +- **Every committed prompt re-renders on its recorded version.** The prompt + sha256s committed in `docs/gpt61sol/` and `docs/haiku55/` (the seed digests + and the judge provenance records of releases 20260929 through 20261010) + re-render from each release's committed board on the version their verdicts + record, v1. This test is local only: it needs the audit grounding, which is + not committed. In CI, a property test checks that `prepare_audit` keeps any + judged case's prompt, verdict and sidecar bytes when the case is unchanged. +- **v2 differs from v1 only by the dropped clause.** The header diff is one + deletion, `V1_REVIEW_CLAIM`. For any case, the two prompts differ only in + their headers, and the v2 prompt is the v1 prompt without that clause. +- **Version selection is explicit, never inferred.** The renderers have no + default version, and every call in `policybench/` and `scripts/` names one. + `prepare_audit` decides whether to keep a verdict from the recorded version + alone: a prompt on any other version is re-opened, never adopted. +- No header is a prefix of another, so a prompt's version is unambiguous. +- `prepare_audit` is idempotent, and the tree it writes validates. + ## Acting on suspect references Cases with `reference_suspect=true` are candidate PolicyEngine or data bugs. diff --git a/policybench/audit.py b/policybench/audit.py index a428c046..55ac34fb 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -36,7 +36,6 @@ from policybench.case_annotations import _format_value, wrong_prediction_rows from policybench.full_run_export import load_case_reference_explanations from policybench.judge_template import ( - CURRENT_TEMPLATE_VERSION, JUDGE_TEMPLATE_HEADERS, TEMPLATE_VERSION_FIELD, recorded_template_version, @@ -285,16 +284,16 @@ def _scenario_questions( # The judge's prompt is a versioned header (policybench.judge_template) and # the case. A verdict is carried only on the exact bytes it was judged on, so -# a seed case renders with the version its sidecar records. +# a seed case renders with the version its sidecar records. Every caller names +# the version: nothing here picks one by default or reads one off a prompt. -def render_case_prompt( - case: AuditCase, template_version: int = CURRENT_TEMPLATE_VERSION -) -> str: +def render_case_prompt(case: AuditCase, *, template_version: int) -> str: """Render the self-contained classification prompt for one case. ``template_version`` picks the header (``JUDGE_TEMPLATE_HEADERS``); the - rest of the prompt does not depend on it. + rest of the prompt does not depend on it. It has no default: a release + reproduces its prompts only on the version its verdicts were judged on. """ lines = [ template_header(template_version), @@ -330,7 +329,8 @@ def prepare_audit( country_dir: Path, audit_dir: Path, grounding_lookup: dict[tuple[str, str], str] | None = None, - template_version: int = CURRENT_TEMPLATE_VERSION, + *, + template_version: int, ) -> list[AuditCase]: """Write per-case prompts, the shared output schema, and a manifest. @@ -346,8 +346,11 @@ def prepare_audit( an earlier run's) is rendered with the template version its sidecar records (absent: v1). If that reproduces its prompt.md, the verdict stands and the prompt keeps its bytes. Otherwise the case changed since it - was judged: the verdict and its sidecar are dropped and the case is - re-opened. New and re-opened cases are rendered with ``template_version``. + was judged, or its version is unknown: the verdict and its sidecar are + dropped and the case is re-opened. New and re-opened cases are rendered + with ``template_version``, which the caller must name. The version a case + renders with comes from the sidecar or the caller, never from prompt.md: + a prompt that begins with another version's header is not adopted. """ template_header(template_version) cases = build_audit_cases(country_dir, grounding_lookup=grounding_lookup) @@ -383,7 +386,7 @@ def prepare_audit( # judged on, so an unchanged case keeps its prompt's bytes. judged_on = recorded_template_version(_sidecar(case_dir)) if judged_on is not None: - seeded = render_case_prompt(case, judged_on) + seeded = render_case_prompt(case, template_version=judged_on) # A verdict without its prompt keeps it, as before # templates were versioned. if not prompt_path.exists() or prompt_path.read_text() == seeded: @@ -399,7 +402,7 @@ def prepare_audit( verdict_path.unlink() (case_dir / "verdict.meta.json").unlink(missing_ok=True) if new_prompt is None: - new_prompt = render_case_prompt(case, template_version) + new_prompt = render_case_prompt(case, template_version=template_version) prompt_path.write_text(new_prompt) return cases @@ -423,9 +426,9 @@ def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: """Each judged case whose prompt and verdict disagree on the template. A tree may mix versions: a release's carried seeds keep the version they - were judged on, while its new and re-opened cases use the current one. - Every verdict must have its prompt.md, its sidecar must name a known - version (absent: v1), and prompt.md must begin with that version's + were judged on, while its new and re-opened cases use the one its driver + names. Every verdict must have its prompt.md, its sidecar must name a + known version (absent: v1), and prompt.md must begin with that version's header. Returns ``(case_id, problem)`` for each case that fails, in case order. """ diff --git a/policybench/cli.py b/policybench/cli.py index f7990fb5..cc474617 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -15,6 +15,10 @@ PROGRAMS, get_programs, ) +from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, +) def _ensure_parent_dir(output_path: str) -> None: @@ -567,9 +571,12 @@ def main(): audit_prepare_parser.add_argument( "--template-version", type=int, - default=None, - help="Judge template version for new and re-opened cases (default: the " - "current one; a judged case keeps the version its verdict records)", + required=True, + help="Judge template version for new and re-opened cases " + "(policybench.judge_template; a new audit uses the current one, " + f"{CURRENT_TEMPLATE_VERSION}). Required: a release reproduces its prompts " + "only on the version its verdicts were judged on. A judged case keeps " + "the version its verdict records", ) audit_collect_parser = subparsers.add_parser( @@ -1377,16 +1384,7 @@ def main(): for row in grounding_df.itertuples() if str(row.grounding).strip() } - from policybench.judge_template import ( - CURRENT_TEMPLATE_VERSION, - JUDGE_TEMPLATE_HEADERS, - ) - - template_version = ( - CURRENT_TEMPLATE_VERSION - if args.template_version is None - else args.template_version - ) + template_version = args.template_version if template_version not in JUDGE_TEMPLATE_HEADERS: raise SystemExit( f"audit-prepare: no judge template version {template_version}; " @@ -1399,7 +1397,8 @@ def main(): template_version=template_version, ) print( - f"Prepared {len(cases)} audit cases under {args.audit_dir}. " + f"Prepared {len(cases)} audit cases under {args.audit_dir}; new and " + f"re-opened cases are on judge template v{template_version}. " f"Run scripts/run_audit_codex.sh {args.audit_dir} to classify." ) diff --git a/policybench/judge_template.py b/policybench/judge_template.py index 41e6d0c6..2b55c773 100644 --- a/policybench/judge_template.py +++ b/policybench/judge_template.py @@ -7,21 +7,29 @@ must still produce every version a carried verdict was judged on. Changing the template means adding a version. -- **v1**, the only template before versions were added (2026-10-09), so - every verdict judged before then, through release dashboard-data-20261006 - and the Claude Haiku 5.5 release's judges. It tells the judge that the +- **v1**, the only template before versions were added, so every verdict + through release dashboard-data-20261010 (Claude Haiku 5.5), the last + release judged before versions existed. It tells the judge that the reference pipeline "has survived an adversarial review program" and that "the few real bugs found were fixed before this run". That is not so: release dashboard-data-20261006's exclusion record stops scoring 28 outputs for engine defects, and the 2026-10-05 reference - adversary found four more. -- **v2**, v1 without that claim. New cases and re-opened cases use it. + adversary confirmed four more. +- **v2**, v1 without that claim: the version a new audit names. Each verdict's provenance sidecar (``verdict.meta.json``) records the version of the prompt it was judged on as ``judge_template_version``. A sidecar without the field, or a verdict without a sidecar, predates the versions and was judged on v1. +Version selection is explicit: every renderer +(:func:`policybench.audit.render_case_prompt`, +:func:`policybench.audit.prepare_audit`, ``policybench audit-prepare``) takes +the version as a required argument, and a judged case renders on the version +its sidecar records. Nothing picks a version by default, and nothing chooses +one by reading a prompt: :func:`template_version_of` only records what a +judge read and checks that a tree agrees with its sidecars. + Dependency-free, so the audit runners can read a prompt's version. """ @@ -33,7 +41,7 @@ TEMPLATE_VERSION_FIELD = "judge_template_version" # The version of a verdict whose sidecar does not record one. UNRECORDED_TEMPLATE_VERSION = 1 -# The version new and re-opened cases are rendered with. +# The newest version: the one a new audit names. No renderer defaults to it. CURRENT_TEMPLATE_VERSION = 2 _V1 = """\ diff --git a/scripts/finish_adds0928.py b/scripts/finish_adds0928.py index 1ea0cbb3..db7b2b8f 100644 --- a/scripts/finish_adds0928.py +++ b/scripts/finish_adds0928.py @@ -47,6 +47,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 # main at release dashboard-data-20260922c, whose committed references are the base. BASE_COMMIT = "3220a7a62b6be83032e9313c9df539c619ad8932" BASE_REFERENCE_SHA256 = { @@ -492,7 +497,12 @@ def prepare_cases(args, bundle) -> None: (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) pending = validate_verdicts(audit, remove_invalid=True) write_json(args.stage_dir / "pending.json", pending) print(f"Prepared audit: {len(pending)} cases need Opus 5.5") diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py index 11551e56..982ae6f6 100644 --- a/scripts/finish_gpt61sol.py +++ b/scripts/finish_gpt61sol.py @@ -53,6 +53,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 ADJUDICATIONS = "us_adjudications.json" # Stage files export binds: the cases GPT-6.1 Sol re-opened, and the # wording-only amendments a developer lists for them. @@ -1414,7 +1419,12 @@ def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) changes = check_prompt_changes(audit, seeded) write_json( args.stage_dir / "prompt-changes.json", diff --git a/scripts/finish_haiku55.py b/scripts/finish_haiku55.py index d8b315b5..233f09ec 100644 --- a/scripts/finish_haiku55.py +++ b/scripts/finish_haiku55.py @@ -69,6 +69,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 ADJUDICATIONS = "us_adjudications.json" # Stage files export binds: the cases Claude Haiku 5.5 re-opened, and the # wording-only amendments a developer lists for them. @@ -2334,7 +2339,12 @@ def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: if (source / name).is_file(): shutil.copyfile(source / name, target / name) lookup = grounding_lookup(args.grounding) - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) changes = check_prompt_changes(audit, seeded) write_json( args.stage_dir / "prompt-changes.json", @@ -4327,6 +4337,9 @@ def render_audit_with( A scratch run directory links every other bundle file and holds the build's, so nothing in the stage changes. The order is prepare_audit's. + Every prompt is on JUDGE_TEMPLATE_VERSION, the version of every verdict + this release's stage holds; install-references refuses a stage whose + prepare_audit rendering differs. """ import tempfile @@ -4350,7 +4363,11 @@ def render_audit_with( with object_strings(): cases = build_audit_cases(run / "us", grounding_lookup=lookup) return [ - (case.case_id, render_case_prompt(case), case.to_manifest_row()) + ( + case.case_id, + render_case_prompt(case, template_version=JUDGE_TEMPLATE_VERSION), + case.to_manifest_row(), + ) for case in cases ] @@ -4545,7 +4562,12 @@ def install_references(args) -> None: for case, reason in reopen.items(): set_aside(audit, case, reason) with object_strings(): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) manifest = [ json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() ] diff --git a/tests/test_audit.py b/tests/test_audit.py index 525dcb91..a52b2a67 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -22,6 +22,7 @@ prepare_audit, render_case_prompt, ) +from policybench.judge_template import CURRENT_TEMPLATE_VERSION @pytest.fixture @@ -87,7 +88,7 @@ def test_build_audit_cases_groups_wrong_by_case(country_dir: Path): def test_render_case_prompt_includes_reference_and_models(country_dir: Path): case = next(c for c in build_audit_cases(country_dir) if c.scenario_id == "s0") - prompt = render_case_prompt(case) + prompt = render_case_prompt(case, template_version=CURRENT_TEMPLATE_VERSION) assert "POLICYENGINE REFERENCE VALUE: $0.00" in prompt assert "m1: answered $250.00" in prompt assert "Used gross income test." in prompt @@ -114,7 +115,9 @@ def test_parse_verdict_tolerates_prose_around_json(tmp_path: Path): def test_prepare_audit_writes_layout(country_dir: Path, tmp_path: Path): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) assert (audit_dir / "schema.json").exists() assert (audit_dir / "cases.jsonl").exists() for case in cases: @@ -127,7 +130,9 @@ def test_collect_audit_folds_verdicts_and_tracks_missing( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) # Write a verdict for the s0 case only; leave s1 missing. s0 = next(c for c in cases if c.scenario_id == "s0") verdict = { @@ -169,7 +174,9 @@ def test_collect_audit_rejects_invalid_failure_source( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") (audit_dir / "cases" / s0.case_id / "verdict.json").write_text( json.dumps( @@ -223,7 +230,9 @@ def test_reference_derivation_reaches_the_prompt(tmp_path: Path): case = build_audit_cases(d)[0] assert "over the income test" in case.reference_derivation - assert "HOW POLICYENGINE DERIVED THE REFERENCE" in render_case_prompt(case) + assert "HOW POLICYENGINE DERIVED THE REFERENCE" in render_case_prompt( + case, template_version=CURRENT_TEMPLATE_VERSION + ) def test_binary_variable_renders_yes_no(tmp_path: Path): @@ -286,7 +295,9 @@ def test_missing_prediction_renders_as_missing(tmp_path: Path): case = next(c for c in build_audit_cases(d) if c.scenario_id == "s0") silent = next(m for m in case.wrong_models if m.model == "silent") assert silent.prediction == "missing" - assert "answered missing" in render_case_prompt(case) + assert "answered missing" in render_case_prompt( + case, template_version=CURRENT_TEMPLATE_VERSION + ) def test_collect_defaults_missing_model_to_case_source( @@ -294,7 +305,9 @@ def test_collect_defaults_missing_model_to_case_source( ): """When the verdict omits a wrong model, its row inherits the case source.""" audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") (audit_dir / "cases" / s0.case_id / "verdict.json").write_text( json.dumps( @@ -372,7 +385,7 @@ def test_collect_empty_wrong_set_keeps_header(tmp_path: Path): ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) out = collect_audit(d, audit_dir) assert list(out["row"].columns) and "failure_source" in out["row"].columns assert list(out["case"].columns) and "case_failure_source" in out["case"].columns @@ -410,7 +423,7 @@ def test_parse_failure_only_case_skips_codex_and_is_deterministic(tmp_path: Path ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) s0 = next(c for c in cases if c.scenario_id == "s0") # No prompt is written for a parse-failure-only case (no Codex call). assert not (audit_dir / "cases" / s0.case_id / "prompt.md").exists() @@ -446,7 +459,7 @@ def test_budget_exhaustion_source_survives_deterministic_missing_audit( ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) out = collect_audit(d, audit_dir) assert out["row"].iloc[0]["failure_source"] == "budget_exhausted_at_ceiling" @@ -494,7 +507,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) s0 = cases[0] verdict_path = audit_dir / "cases" / s0.case_id / "verdict.json" verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') @@ -514,7 +527,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): } ] ).to_csv(d / "predictions.csv", index=False) - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) assert not verdict_path.exists() # stale verdict dropped assert not meta_path.exists() # and its provenance sidecar with it @@ -522,12 +535,12 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): # (the re-opened case's prompt is the current version's). verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') meta_path.write_text(json.dumps({"judge_template_version": 2})) - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) assert verdict_path.exists() # A verdict whose sidecar records no version was judged on v1, so the # current version's prompt is not the one it was judged on. meta_path.write_text('{"judge_model_requested": "opus"}') - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) assert not verdict_path.exists() assert not meta_path.exists() @@ -553,7 +566,7 @@ def test_collect_audit_coerces_parse_source_for_parsed_predictions(tmp_path: Pat ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) case = cases[0] verdict_path = audit_dir / "cases" / case.case_id / "verdict.json" verdict_path.write_text( @@ -619,7 +632,9 @@ def test_collect_uses_per_model_diagnosis_for_row_annotation( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") verdict = make_verdict( "Applied the net-income allotment formula without the earned income " @@ -645,7 +660,9 @@ def test_collect_flags_hedged_verdicts_for_rejudging(country_dir: Path, tmp_path # The shipped 20260707c failure class: the judge answers "is the # reference wrong?" instead of diagnosing the model. audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") verdict = make_verdict( "The reference is plausible under the stated PolicyEngine " @@ -684,7 +701,7 @@ def test_prompt_treats_reference_as_verified_and_demands_diagnosis( country_dir: Path, ): case = next(c for c in build_audit_cases(country_dir) if c.scenario_id == "s0") - prompt = render_case_prompt(case) + prompt = render_case_prompt(case, template_version=CURRENT_TEMPLATE_VERSION) assert "Treat the reference and its derivation as correct" in prompt assert "diagnosis" in prompt assert "mechanically rejected" in prompt @@ -696,9 +713,15 @@ def test_grounding_lookup_renders_engine_facts_block(country_dir: Path): cases = build_audit_cases(country_dir, grounding_lookup=lookup) grounded = next(c for c in cases if c.scenario_id == "s0") bare = next(c for c in cases if c.scenario_id == "s1") - assert "AUTHORITATIVE ENGINE FACTS" in render_case_prompt(grounded) - assert "SENIOR_OR_DISABLED" in render_case_prompt(grounded) - assert "AUTHORITATIVE ENGINE FACTS" not in render_case_prompt(bare) + assert "AUTHORITATIVE ENGINE FACTS" in render_case_prompt( + grounded, template_version=CURRENT_TEMPLATE_VERSION + ) + assert "SENIOR_OR_DISABLED" in render_case_prompt( + grounded, template_version=CURRENT_TEMPLATE_VERSION + ) + assert "AUTHORITATIVE ENGINE FACTS" not in render_case_prompt( + bare, template_version=CURRENT_TEMPLATE_VERSION + ) def test_schema_requires_per_model_diagnosis(): diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index cb8cf185..2ed39158 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -34,6 +34,7 @@ ) from policybench.judge_template import ( # noqa: E402 CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, V1_REVIEW_CLAIM, template_version_of, ) @@ -1006,9 +1007,16 @@ def prepare(new_rows, *, grounding_path=grounding): return seed, stage, prepare +@pytest.mark.parametrize( + "version", sorted({driver.JUDGE_TEMPLATE_VERSION, CURRENT_TEMPLATE_VERSION}) +) def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( - seeded_stage, + seeded_stage, monkeypatch, version ): + """The driver re-opens the case the new model joins on the version it + names: v1 as committed, or v2 as a new release's driver names it. The + other case keeps the seed's bytes on the v1 template it was judged on.""" + monkeypatch.setattr(driver, "JUDGE_TEMPLATE_VERSION", version) seed, stage, prepare = seeded_stage audit = prepare( [ @@ -1025,15 +1033,13 @@ def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( assert (audit / "cases" / untouched / name).read_bytes() == ( seed / "cases" / untouched / name ).read_bytes() - # The kept case stays on the v1 template it was judged on; the re-opened - # one moves to the current template. assert ( template_version_of((audit / "cases" / untouched / "prompt.md").read_bytes()) == 1 ) assert ( template_version_of((audit / "cases" / joined / "prompt.md").read_bytes()) - == CURRENT_TEMPLATE_VERSION + == version ) assert json.loads((stage / "prompt-changes.json").read_text()) == { "added": [], @@ -1042,14 +1048,12 @@ def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( } wrong = ["m1", "m2", NEW] write_verdict( - audit / "cases" / joined, - _verdict(wrong), - judge_template_version=CURRENT_TEMPLATE_VERSION, + audit / "cases" / joined, _verdict(wrong), judge_template_version=version ) seed_binding = driver.load_seed(stage) assert seed_binding == driver.seed_digest(seed) assert driver.validate_verdicts(audit, seed=seed_binding) == [] - # The stage mixes a v1 seed and a current-template verdict, and validates. + # The stage validates; on v2 it mixes a v1 seed and a v2 verdict. assert template_version_problems(audit) == [] @@ -1443,8 +1447,8 @@ def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): So check_prompt_changes may attribute every changed or new prompt in a stage to GPT-6.1 Sol joining its case. #182's review excluded scenario_023 head_medicaid_eligible and rewrote its adjudication and case - note; neither enters a prompt, and its prompt is unchanged. On the current - template every prompt is the seed's without v1's review claim. + note; neither enters a prompt, and its prompt is unchanged. On v2 every + prompt is the seed's without v1's review claim. """ if not (SEED.is_dir() and GROUNDING.is_file()): pytest.skip("release 20260929's audit or grounding is not on this machine") @@ -1459,7 +1463,7 @@ def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): meta = json.loads((SEED / "cases" / case / "verdict.meta.json").read_text()) assert rendered[case] == meta["prompt_sha256"] current = tmp_path / "current" - _prepare_like_the_driver(country_dir, current) + _prepare_like_the_driver(country_dir, current, template_version=2) for case_dir in sorted((SEED / "cases").iterdir()): seed_prompt = (case_dir / "prompt.md").read_text() assert (current / "cases" / case_dir.name / "prompt.md").read_text() == ( @@ -1489,8 +1493,8 @@ def test_every_frozen_verdict_rerenders_on_its_recorded_template(tmp_path, relea """Local only: copied into a stage as prepare_cases copies a seed, every verdict of a frozen audit tree re-renders on the template version its sidecar records (absent: v1) to the exact prompt it was judged on, so - prepare_audit keeps every prompt, verdict and sidecar byte for byte under - the current default; and the tree validates.""" + prepare_audit keeps every prompt, verdict and sidecar byte for byte + whichever version its caller names; and the tree validates.""" from policybench.audit import template_version_problems from policybench.judge_template import recorded_template_version @@ -1523,12 +1527,15 @@ def test_every_frozen_verdict_rerenders_on_its_recorded_template(tmp_path, relea for case, files in judged.items() } assert set(versions.values()) == {1} - _prepare_like_the_driver(country_dir, stage) - assert (stage / "cases.jsonl").read_bytes() == (tree / "cases.jsonl").read_bytes() - for case, files in judged.items(): - for name, data in files.items(): - assert (stage / "cases" / case / name).read_bytes() == data, (case, name) - assert template_version_problems(stage) == [] + for version in sorted(JUDGE_TEMPLATE_HEADERS): + _prepare_like_the_driver(country_dir, stage, template_version=version) + cases_jsonl = (stage / "cases.jsonl").read_bytes() + assert cases_jsonl == (tree / "cases.jsonl").read_bytes(), version + for case, files in judged.items(): + for name, data in files.items(): + path = stage / "cases" / case / name + assert path.read_bytes() == data, (version, case, name) + assert template_version_problems(stage) == [] def test_check_prompt_changes_names_new_incumbent_only_cases(tmp_path): diff --git a/tests/test_finish_haiku55.py b/tests/test_finish_haiku55.py index 8d77f211..4f798b1f 100644 --- a/tests/test_finish_haiku55.py +++ b/tests/test_finish_haiku55.py @@ -1148,7 +1148,9 @@ def seeded_stage(tmp_path, monkeypatch): lookup = {("s1", "snap"): "Gross test: pass."} seed = tmp_path / "seed" seed_board = _board(tmp_path / "seed-board", INCUMBENT_ROWS) - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + # Release 20261006's seed was judged on judge template v1, before versions + # existed, so its sidecars record none. + prepare_audit(seed_board / "us", seed, grounding_lookup=lookup, template_version=1) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], @@ -1965,7 +1967,8 @@ def test_each_20261006_entry_names_its_seed_verdict_as_its_sidecar_records_it(): @pytest.mark.slow def test_every_seed_prompt_rerenders_except_the_nine_reworded(tmp_path): """Local only: release 20261006's 46-model board, as BASE_COMMIT holds it, - renders exactly the seed's cases, and every seed prompt except the nine + renders exactly the seed's cases, and on judge template v1, the template + the seed was judged on, every seed prompt except the nine REWORDED_SINCE_SEED, whose reference explanations it reworded (this reads the 43 MB predictions from git into scratch). So check_prompt_changes may attribute every other changed or new prompt to @@ -1996,7 +1999,8 @@ def test_every_seed_prompt_rerenders_except_the_nine_reworded(tmp_path): if arrow else contextlib.nullcontext() ): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + # The seed was judged on judge template v1. + prepare_audit(bundle / "us", audit, grounding_lookup=lookup, template_version=1) # The manifest records each case's reference derivation, so it differs # from the seed's in that field of the nine cases and nowhere else. ours, seeds = ( @@ -6140,8 +6144,11 @@ def patch(name, value): lookup = {("s1", "snap"): "Gross test: pass."} seed = root / "seed" seed_board = _board(root / "seed-board", INCUMBENT_ROWS) + # The seed is on judge template v1, as release 20261006's was. with driver.object_strings(): - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + prepare_audit( + seed_board / "us", seed, grounding_lookup=lookup, template_version=1 + ) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py index 86816746..8c757d6e 100644 --- a/tests/test_judge_template.py +++ b/tests/test_judge_template.py @@ -2,13 +2,21 @@ the version each was judged on. Invariants: +- every committed prompt re-renders on its recorded version: the prompt + sha256s that docs/gpt61sol/ and docs/haiku55/ commit re-render from each + release's committed board on the version their verdicts record (local only, + for it needs the audit grounding), and prepare_audit keeps a judged case's + prompt, verdict and sidecar bytes when the case re-renders to them with its + recorded version (absent: v1); +- v2 differs from v1 only by the dropped clause: the header diff is one + deletion, V1_REVIEW_CLAIM, and the rest of a prompt does not depend on the + version; +- version selection is explicit, never inferred: render_case_prompt, + prepare_audit and audit-prepare require a version, every call names one, + and prepare_audit keeps a verdict on its recorded version alone, never on + the version a prompt begins with; - a version's header never changes (pinned by sha256), and no header is a prefix of another, so a prompt's version is read off its bytes; -- the rest of a prompt does not depend on the version, and v2 is v1 without - the review claim; -- prepare_audit keeps a judged case's prompt, verdict and sidecar bytes when - the case re-renders to them with its recorded version (absent: v1), and - renders every other case with the requested version (default: current); - after prepare_audit, a tree validates (template_version_problems is empty), and a second prepare_audit changes nothing. """ @@ -16,8 +24,13 @@ from __future__ import annotations import ast +import contextlib +import difflib +import functools import hashlib +import inspect import json +import os import subprocess import sys import tempfile @@ -116,6 +129,23 @@ def test_v2_is_v1_without_the_review_claim(): ) +def test_the_header_diff_is_one_deletion_of_the_clause(): + """v2 differs from v1 only by the dropped clause, and no diff can place + the deletion anywhere else: the common prefix ends where the clause + starts, the common suffix starts where it ends, and difflib's edit is that + one deletion.""" + v1, v2 = JUDGE_TEMPLATE_HEADERS[1], JUDGE_TEMPLATE_HEADERS[2] + prefix = len(os.path.commonprefix([v1, v2])) + suffix = len(os.path.commonprefix([v1[::-1], v2[::-1]])) + assert prefix + suffix == len(v2) + assert v1[prefix : len(v1) - suffix] == V1_REVIEW_CLAIM + opcodes = difflib.SequenceMatcher(None, v1, v2, autojunk=False).get_opcodes() + edits = [ + (tag, v1[i1:i2], v2[j1:j2]) for tag, i1, i2, j1, j2 in opcodes if tag != "equal" + ] + assert edits == [("delete", V1_REVIEW_CLAIM, "")] + + def test_no_header_is_a_prefix_of_another(): for version, header in JUDGE_TEMPLATE_HEADERS.items(): for other, other_header in JUDGE_TEMPLATE_HEADERS.items(): @@ -190,7 +220,7 @@ def audit_cases(draw) -> AuditCase: @given(audit_cases()) def test_a_prompts_version_is_read_off_its_bytes(case): for version in JUDGE_TEMPLATE_HEADERS: - prompt = render_case_prompt(case, version) + prompt = render_case_prompt(case, template_version=version) assert prompt.startswith(template_header(version)) assert template_version_of(prompt) == version assert template_version_of(prompt.encode("utf-8")) == version @@ -200,15 +230,17 @@ def test_a_prompts_version_is_read_off_its_bytes(case): @given(audit_cases()) def test_only_the_header_depends_on_the_version(case): bodies = { - render_case_prompt(case, version)[len(header) :] + render_case_prompt(case, template_version=version)[len(header) :] for version, header in JUDGE_TEMPLATE_HEADERS.items() } assert len(bodies) == 1 - v1, v2 = render_case_prompt(case, 1), render_case_prompt(case, 2) + v1 = render_case_prompt(case, template_version=1) + v2 = render_case_prompt(case, template_version=2) assert v2 == v1.replace(V1_REVIEW_CLAIM, "", 1) - assert render_case_prompt(case) == render_case_prompt( - case, CURRENT_TEMPLATE_VERSION - ) + # The one deletion is the clause, where the header states it. + start = v1.index(V1_REVIEW_CLAIM) + assert v2 == v1[:start] + v1[start + len(V1_REVIEW_CLAIM) :] + assert start == JUDGE_TEMPLATE_HEADERS[1].index(V1_REVIEW_CLAIM) @PROPERTY @@ -301,12 +333,14 @@ def _judge(case_dir: Path, sidecar: dict | None) -> None: def _seed(audit: Path, board: Path, states: dict[str, str]) -> dict[str, AuditCase]: """Prepare ``board`` into ``audit``, then give each scenario's case the prompt and verdict its state names. Returns the cases by scenario.""" - prepare_audit(board, audit) + prepare_audit(board, audit, template_version=CURRENT_TEMPLATE_VERSION) cases = {case.scenario_id: case for case in build_audit_cases(board)} for scenario, state in states.items(): case_dir = audit / "cases" / cases[scenario].case_id (case_dir / "prompt.md").write_text( - render_case_prompt(cases[scenario], SEED_PROMPT_VERSION[state]) + render_case_prompt( + cases[scenario], template_version=SEED_PROMPT_VERSION[state] + ) ) if state != "unjudged": _judge(case_dir, SEED_STATES[state]) @@ -366,17 +400,17 @@ def test_prepare_keeps_each_judged_case_on_its_own_template(plan, version): assert f"{case_dir}/verdict.json" not in after, state assert f"{case_dir}/verdict.meta.json" not in after, state assert after[f"{case_dir}/prompt.md"] == render_case_prompt( - cases[scenario], version + cases[scenario], template_version=version ).encode("utf-8") assert template_version_problems(audit) == [] prepare_audit(board, audit, template_version=version) assert _tree(audit) == after -def test_new_cases_get_the_current_version(tmp_path): +def test_a_new_audit_on_the_current_version_drops_the_claim(tmp_path): board = _board(tmp_path / "us", {"s0": 250.0, "s1": 100.0}) audit = tmp_path / "audit" - cases = prepare_audit(board, audit) + cases = prepare_audit(board, audit, template_version=CURRENT_TEMPLATE_VERSION) for case in cases: prompt = (audit / "cases" / case.case_id / "prompt.md").read_bytes() assert template_version_of(prompt) == CURRENT_TEMPLATE_VERSION == 2 @@ -391,7 +425,7 @@ def test_a_requested_version_renders_new_cases_and_an_unknown_one_is_refused( audit = tmp_path / "audit" (case,) = prepare_audit(board, audit, template_version=1) prompt = (audit / "cases" / case.case_id / "prompt.md").read_text() - assert prompt == render_case_prompt(case, 1) + assert prompt == render_case_prompt(case, template_version=1) assert "fixed before this run" in prompt with pytest.raises(ValueError, match="no judge template version 3"): prepare_audit(board, tmp_path / "other", template_version=3) @@ -399,9 +433,9 @@ def test_a_requested_version_renders_new_cases_and_an_unknown_one_is_refused( def test_a_v1_seed_carries_over_and_a_reopened_case_moves_to_v2(tmp_path): - """The fold drivers' flow: a seed judged on v1 (its sidecar predates the - field) is copied into a stage; the case a new model joins is re-opened on - v2, and the other keeps the seed's bytes.""" + """The fold drivers' flow, for a driver that names v2: a seed judged on v1 + (its sidecar predates the field) is copied into a stage; the case a new + model joins is re-opened on v2, and the other keeps the seed's bytes.""" seed_board = _board(tmp_path / "seed" / "us", {"s0": 250.0, "s1": 100.0}) seed = tmp_path / "seed" / "audit" prepare_audit(seed_board, seed, template_version=1) @@ -420,7 +454,10 @@ def test_a_v1_seed_carries_over_and_a_reopened_case_moves_to_v2(tmp_path): new = predictions.assign(model="new") new.loc[new.scenario_id == "s1", "prediction"] = 0.0 pd.concat([predictions, new]).to_csv(board / "predictions.csv", index=False) - cases = {case.scenario_id: case for case in prepare_audit(board, stage)} + cases = { + case.scenario_id: case + for case in prepare_audit(board, stage, template_version=2) + } kept, reopened = (stage / "cases" / cases[s].case_id for s in ("s1", "s0")) for name in ("prompt.md", "verdict.json", "verdict.meta.json"): assert (kept / name).read_bytes() == ( @@ -428,7 +465,9 @@ def test_a_v1_seed_carries_over_and_a_reopened_case_moves_to_v2(tmp_path): ).read_bytes() assert template_version_of((kept / "prompt.md").read_bytes()) == 1 assert not (reopened / "verdict.json").exists() - assert (reopened / "prompt.md").read_text() == render_case_prompt(cases["s0"], 2) + assert (reopened / "prompt.md").read_text() == render_case_prompt( + cases["s0"], template_version=2 + ) assert template_version_problems(stage) == [] @@ -460,9 +499,20 @@ def test_a_mixed_tree_validates_and_collects(tmp_path): assert list(out["template"].columns) == ["case_id", "problem"] assert sorted(out["missing"]["case_id"]) == [cases["s4"].case_id] assert len(out["case"]) == 4 - # Preparing it again keeps every verdict: each case renders to its bytes. + # Preparing it again, on either version, keeps every verdict: each judged + # case renders to its bytes on the version it records. Only the unjudged + # case follows the version the caller names. before = _tree(audit) - prepare_audit(board, audit) + unjudged = f"cases/{cases['s4'].case_id}/prompt.md" + for version in sorted(JUDGE_TEMPLATE_HEADERS): + prepare_audit(board, audit, template_version=version) + after = _tree(audit) + assert after[unjudged] == render_case_prompt( + cases["s4"], template_version=version + ).encode("utf-8") + assert {k: v for k, v in after.items() if k != unjudged} == { + k: v for k, v in before.items() if k != unjudged + } assert _tree(audit) == before _collect_cli(board, audit, tmp_path / "out") assert (tmp_path / "out" / "us_audit_case_annotations.csv").is_file() @@ -504,16 +554,16 @@ def meta(scenario: str, **fields) -> None: assert not (tmp_path / "out").exists() # Preparing again re-opens every flagged case except the verdict that # lost its prompt, which keeps its verdict and gets its v1 prompt back. - prepare_audit(board, audit) + prepare_audit(board, audit, template_version=2) assert template_version_problems(audit) == [] assert (case_dir["s0"] / "verdict.json").is_file() assert (case_dir["s0"] / "prompt.md").read_text() == render_case_prompt( - cases["s0"], 1 + cases["s0"], template_version=1 ) for scenario in ("s1", "s2", "s3", "s4"): assert not (case_dir[scenario] / "verdict.json").exists(), scenario assert (case_dir[scenario] / "prompt.md").read_text() == render_case_prompt( - cases[scenario], 2 + cases[scenario], template_version=2 ) @@ -526,20 +576,369 @@ def _collect_cli(board: Path, audit: Path, out: Path) -> None: cli.main() -def test_audit_prepare_takes_a_template_version(tmp_path, capsys): +def test_audit_prepare_requires_a_template_version(tmp_path, capsys): from policybench import cli board = _board(tmp_path / "us", {"s0": 250.0}) - for flag, version in ((["--template-version", "1"], 1), ([], 2)): + for version in sorted(JUDGE_TEMPLATE_HEADERS): audit = tmp_path / f"audit-{version}" argv = ["policybench", "audit-prepare", "--country-dir", str(board)] - argv += ["--audit-dir", str(audit), *flag] + argv += ["--audit-dir", str(audit), "--template-version", str(version)] with mock.patch.object(sys, "argv", argv): cli.main() (prompt,) = (audit / "cases").glob("*/prompt.md") assert template_version_of(prompt.read_bytes()) == version + assert f"on judge template v{version}." in capsys.readouterr().out + # No flag: argparse refuses before anything is written. + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(tmp_path / "audit-none")] + with mock.patch.object(sys, "argv", argv): + with pytest.raises(SystemExit) as refused: + cli.main() + assert refused.value.code == 2 + assert "--template-version" in capsys.readouterr().err + assert not (tmp_path / "audit-none").exists() argv = ["policybench", "audit-prepare", "--country-dir", str(board)] argv += ["--audit-dir", str(tmp_path / "audit-9"), "--template-version", "9"] with mock.patch.object(sys, "argv", argv): with pytest.raises(SystemExit, match="no judge template version 9"): cli.main() + assert not (tmp_path / "audit-9").exists() + + +# --- Version selection is explicit, never inferred ------------------------------ + + +def test_the_renderers_have_no_default_version(): + """Every renderer takes the version as a required keyword: none falls back + to CURRENT_TEMPLATE_VERSION, so adding a version moves no caller.""" + for function in (render_case_prompt, prepare_audit): + parameter = inspect.signature(function).parameters["template_version"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY, function.__name__ + assert parameter.default is inspect.Parameter.empty, function.__name__ + + +def test_a_renderer_called_without_a_version_is_refused(tmp_path): + board = _board(tmp_path / "us", {"s0": 250.0}) + (case,) = build_audit_cases(board) + with pytest.raises(TypeError, match="template_version"): + render_case_prompt(case) + with pytest.raises(TypeError): + render_case_prompt(case, CURRENT_TEMPLATE_VERSION) # not by name + with pytest.raises(TypeError, match="template_version"): + prepare_audit(board, tmp_path / "audit") + assert not (tmp_path / "audit").exists() + + +# The modules outside the tests that render judge prompts, and their calls. +RENDERER_CALLS = { + "policybench/audit.py": {"render_case_prompt": 2}, + "policybench/cli.py": {"prepare_audit": 1}, + "scripts/finish_adds0928.py": {"prepare_audit": 1}, + "scripts/finish_gpt61sol.py": {"prepare_audit": 1}, + "scripts/finish_haiku55.py": {"prepare_audit": 2, "render_case_prompt": 1}, +} +RENDERERS = ("render_case_prompt", "prepare_audit") + + +def _called_name(call: ast.Call) -> str | None: + func = call.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def test_every_renderer_call_names_its_version(): + """Static: every call of render_case_prompt or prepare_audit in + policybench/, scripts/ and reference_audit/ passes template_version by + name, and no value passed is read off a prompt (template_version_of).""" + found: dict[str, dict[str, int]] = {} + paths = [ + path + for part in ("policybench", "scripts", "reference_audit") + for path in sorted((ROOT / part).rglob("*.py")) + ] + for path in paths: + relative = path.relative_to(ROOT).as_posix() + for call in ast.walk(ast.parse(path.read_text(), filename=relative)): + if not isinstance(call, ast.Call) or _called_name(call) not in RENDERERS: + continue + name = _called_name(call) + counts = found.setdefault(relative, {}) + counts[name] = counts.get(name, 0) + 1 + keywords = {k.arg: k.value for k in call.keywords} + where = f"{relative}:{call.lineno} {name}" + assert "template_version" in keywords, f"{where} names no version" + inferred = [ + node + for node in ast.walk(keywords["template_version"]) + if isinstance(node, ast.Call) + and _called_name(node) == "template_version_of" + ] + assert not inferred, f"{where} reads its version off a prompt" + for relative, counts in RENDERER_CALLS.items(): + assert found.get(relative) == counts, relative + + +# The release drivers of releases judged before versions existed: each +# judged every verdict on v1. +V1_DRIVERS = ( + "scripts/finish_adds0928.py", + "scripts/finish_gpt61sol.py", + "scripts/finish_haiku55.py", +) + + +def test_each_past_release_driver_renders_on_v1_by_name(): + """Each driver states the version its release was judged on as the + module constant JUDGE_TEMPLATE_VERSION = 1, and passes that constant, not + a literal or another value, to every renderer it calls.""" + for relative in V1_DRIVERS: + tree = ast.parse((ROOT / relative).read_text(), filename=relative) + (value,) = [ + node.value + for node in tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] + == ["JUDGE_TEMPLATE_VERSION"] + ] + assert isinstance(value, ast.Constant) and value.value == 1, relative + calls = [ + call + for call in ast.walk(tree) + if isinstance(call, ast.Call) and _called_name(call) in RENDERERS + ] + assert calls, relative + for call in calls: + (named,) = [k.value for k in call.keywords if k.arg == "template_version"] + assert isinstance(named, ast.Name), (relative, call.lineno) + assert named.id == "JUDGE_TEMPLATE_VERSION", (relative, call.lineno) + + +# A judged case's prompt.md, as the property draws it. +PROMPT_KINDS = ("v1", "v2", "v1 + text", "v2 + text", "no header") +JUDGED_STATES = sorted(state for state in SEED_STATES if state != "unjudged") + + +@PROPERTY +@given( + st.sampled_from(JUDGED_STATES), + st.sampled_from(PROMPT_KINDS), + st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), + TEXT, +) +def test_a_verdict_is_kept_on_its_recorded_version_alone(state, kind, version, text): + """Never inferred: prepare_audit keeps an unchanged judged case's verdict + exactly when prompt.md is the case rendered on the version its sidecar + records. A prompt.md on another version is re-opened, though + template_version_of reads a version off it and though the caller may name + that very version; a re-opened case renders on the caller's version.""" + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + board = _board(root / "us", {"s0": 250.0}) + audit = root / "audit" + (case,) = prepare_audit(board, audit, template_version=version) + case_dir = audit / "cases" / case.case_id + if kind == "no header": + prompt = f"Classify this miss.\n{text}" + else: + prompt = render_case_prompt(case, template_version=int(kind[1])) + if kind.endswith("+ text"): + prompt += f"\n{text}." + (case_dir / "prompt.md").write_text(prompt) + _judge(case_dir, SEED_STATES[state]) + before = _tree(audit) + recorded = recorded_template_version(SEED_STATES[state]) + kept = recorded is not None and prompt == render_case_prompt( + case, template_version=recorded + ) + prepare_audit(board, audit, template_version=version) + after = _tree(audit) + names = ("prompt.md", "verdict.json", "verdict.meta.json") + if kept: + for name in names: + key = f"cases/{case.case_id}/{name}" + assert after.get(key) == before.get(key), (state, kind, name) + else: + assert not (case_dir / "verdict.json").exists(), (state, kind) + assert not (case_dir / "verdict.meta.json").exists(), (state, kind) + assert (case_dir / "prompt.md").read_text() == render_case_prompt( + case, template_version=version + ) + assert template_version_problems(audit) == [] + + +# --- Every committed prompt re-renders on its recorded version ------------------ + +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +BOARD_PATH = Path("paper/snapshot/20260501/runs") / RUN_NAME +ANNOTATIONS_PATH = Path("annotations") / RUN_NAME +# The board files an audit renders from, and the annotations beside them. +BOARD_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", + "predictions.csv.gz", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", + "us_adjudications.json", +) +# Each release PR's squash on main, which commits the board the release's +# audit renders from. A dashboard-data tag marks main when its GitHub release +# was cut, before that merge, so it does not hold the release's board. +RELEASE_COMMITS = { + "20260929": "d616e67c33b6f80dabf5cb7329f069f9a1de069d", + "20260930": "8b4c0ca146bb6f66deba6ce24009d49d70d92df2", + "20261010": "5a8164a001efb27fa55f47fe7ea26666a0de31f8", +} +# The audit grounding these releases rendered with, which the release drivers +# pin as GROUNDING_SHA256. It is not committed, so rendering is local only. +GROUNDING = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/" + "grounding.csv" +) +GROUNDING_SHA256 = "b1e4a9bc74d762f410524a147efcda7d705c3afcfa3dc27f720fa60c54a7b55c" + + +def _seed_records(path: str) -> dict[str, dict]: + """A committed seed digest: each judged case's prompt and verdict sha256.""" + lines = (ROOT / path).read_text().splitlines() + assert lines[0] == "case_id,prompt_sha256,verdict_sha256" + return { + case: {"prompt_sha256": prompt, "verdict_sha256": verdict} + for case, prompt, verdict in (line.split(",") for line in lines[1:]) + } + + +def _provenance_records(path: str) -> dict[str, dict]: + """A committed judge provenance record: each re-judged verdict's entry.""" + verdicts = json.loads((ROOT / path).read_text())["verdicts"] + records = {entry["case_id"]: entry for entry in verdicts} + assert len(records) == len(verdicts), path + return records + + +def _committed_prompts() -> dict[str, dict[str, dict]]: + """Every committed prompt record, by the release whose audit holds it. + + Release 20260929's audit is docs/gpt61sol/seed_digest.csv. Release + 20260930 re-judged the cases docs/gpt61sol/judge_provenance.json lists + and carried the rest; release 20261006 carried that audit, as + docs/haiku55/seed_digest.csv records. Release 20261010 re-judged the + cases docs/haiku55/judge_provenance.json lists and carried the rest. + """ + seed_0929 = _seed_records("docs/gpt61sol/seed_digest.csv") + seed_1006 = _seed_records("docs/haiku55/seed_digest.csv") + return { + "20260929": seed_0929, + "20260930": { + **seed_0929, + **_provenance_records("docs/gpt61sol/judge_provenance.json"), + }, + "20261006": seed_1006, + "20261010": { + **seed_1006, + **_provenance_records("docs/haiku55/judge_provenance.json"), + }, + } + + +def test_the_committed_prompt_records_agree_and_name_a_version(): + """Runs anywhere: each committed record names a known template version + (none records the field, so each records v1), and release 20261006's + seed digest is release 20260930's audit: release 20260929's digest with + the prompts and verdicts of the cases release 20260930 re-judged.""" + committed = _committed_prompts() + assert {release: len(records) for release, records in committed.items()} == { + "20260929": 674, + "20260930": 674, + "20261006": 674, + # The seed's 674 and three cases release 20261010 added. + "20261010": 677, + } + for records in committed.values(): + for case, record in records.items(): + assert recorded_template_version(record) == 1, case + digests = { + release: { + case: (record["prompt_sha256"], record["verdict_sha256"]) + for case, record in records.items() + } + for release, records in committed.items() + } + assert digests["20261006"] == digests["20260930"] + + +@functools.cache +def _blob(commit: str, path: Path) -> bytes: + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path.as_posix()}"], + capture_output=True, + ) + if result.returncode: + pytest.fail( + f"cannot read {path} at {commit[:12]}; fetch full history (git fetch " + f"--unshallow): {result.stderr.decode().strip()}" + ) + return result.stdout + + +@contextlib.contextmanager +def _object_strings(): + """Render as the release drivers did: with pandas' inferred Arrow strings + off.""" + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + with pd.option_context("future.infer_string", False): + yield + else: + yield + + +@pytest.mark.slow +@pytest.mark.parametrize("release", sorted(RELEASE_COMMITS)) +def test_every_committed_prompt_rerenders_on_its_recorded_version(tmp_path, release): + """Local only: release ``release``'s board, read from the commit that + committed it, renders every judged case of its audit, on the version its + committed record names, to the prompt sha256 that record commits; and it + renders no other case.""" + if not GROUNDING.is_file(): + pytest.skip("the audit grounding is not on this machine") + assert hashlib.sha256(GROUNDING.read_bytes()).hexdigest() == GROUNDING_SHA256 + commit = RELEASE_COMMITS[release] + country_dir = tmp_path / RUN_NAME / "us" + country_dir.mkdir(parents=True) + for name in BOARD_FILES: + (country_dir / name).write_bytes(_blob(commit, BOARD_PATH / name)) + (tmp_path / RUN_NAME / "annotations").mkdir() + for name in ANNOTATION_FILES: + (tmp_path / RUN_NAME / "annotations" / name).write_bytes( + _blob(commit, ANNOTATIONS_PATH / name) + ) + grounding = pd.read_csv(GROUNDING) + lookup = { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + committed = _committed_prompts()[release] + rendered = {} + with _object_strings(): + for case in build_audit_cases(country_dir, grounding_lookup=lookup): + if case.to_manifest_row()["parse_failure_only"]: + continue + record = committed.get(case.case_id) + assert record is not None, f"{case.case_id} has no committed prompt" + prompt = render_case_prompt( + case, template_version=recorded_template_version(record) + ) + rendered[case.case_id] = hashlib.sha256(prompt.encode()).hexdigest() + expected = {case: record["prompt_sha256"] for case, record in committed.items()} + assert sorted(rendered) == sorted(expected) + differ = sorted(case for case in rendered if rendered[case] != expected[case]) + assert not differ, differ From 575f25744ba52c66af69252ec647f2b3aa5e15f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 06:54:39 -0400 Subject: [PATCH 3/8] Keep a verdict only on the exact bytes its judge read Answers review round 1 of #210: - prepare_audit compares prompt.md as UTF-8 bytes, writes prompts as bytes and leaves a kept case's files untouched (read_text folded \r and CRLF, so a correct verdict could be dropped and a CRLF prompt kept). - A verdict without prompt.md stands only when its sidecar's prompt_sha256 is the case rendered on its recorded version; the prompt is restored. A sidecar hash that is not prompt.md's bytes re-opens the case. - run_audit_codex.sh has Codex read a private copy of prompt.md, records that copy's prompt_sha256 and template version, and publishes nothing if prompt.md changed while Codex judged it. - Tests: byte-exact properties with \r and CRLF, the missing-prompt cases (the review's changed-case scenario included), a Codex race test, and an allowlist of the one function that reads a version off a prompt. - Docs: each committed prompt re-renders from the board of the release that judged it; release 20261006 judged nothing. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 42 ++--- policybench/audit.py | 89 +++++++---- scripts/run_audit_codex.sh | 41 ++++- tests/test_judge_provenance.py | 45 ++++++ tests/test_judge_template.py | 269 ++++++++++++++++++++++++++++----- 5 files changed, 388 insertions(+), 98 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index 6c3c2f85..f0ab7d4d 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -71,20 +71,24 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` judged case renders on the version its sidecar records. Nothing reads a version off a prompt to choose how to render it: `template_version_of` only records what a judge read and checks a tree. -- Both runners record the version a verdict was judged on in its - `verdict.meta.json` as `judge_template_version`. The Claude runner reads it - from the judged copy of `prompt.md`, the Codex runner from `prompt.md`. A - prompt that begins with no version's header, one `audit-prepare` did not - write, records `null`. +- Both runners have the judge read a private copy of `prompt.md`, and record + that copy's sha256 (`prompt_sha256`) and version (`judge_template_version`) + in `verdict.meta.json`. A verdict is not published if `prompt.md` no longer + holds those bytes. A prompt that begins with no version's header, one + `audit-prepare` did not write, records `null`. - A sidecar without the field, or a verdict without a sidecar, was judged on v1 (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars written before versions existed, not a guess from the prompt. - `audit-prepare` renders a case that already has a verdict with the version - its sidecar records. If that reproduces `prompt.md`, the verdict stands and - the prompt keeps its bytes. Otherwise the case changed since it was judged, - or its version is unknown: the verdict and its sidecar are dropped and the - case is re-opened. A `prompt.md` on another version is re-opened, not - adopted. New and re-opened cases use the version the caller names. + its sidecar records. The verdict stands only if that reproduces + `prompt.md` byte for byte and, when the sidecar records `prompt_sha256` + (the hash of the bytes its judge read), matches it. Without `prompt.md`, + the verdict stands only if that hash matches, and the prompt is restored. + A kept case's files are not rewritten. Otherwise the case changed since it + was judged, or its version or judged bytes are unknown: the verdict and + its sidecar are dropped and the case is re-opened. A `prompt.md` on + another version is re-opened, not adopted. New and re-opened cases use the + version the caller names. - Each release driver names the version its new and re-opened cases render on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, @@ -109,13 +113,17 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` `tests/test_judge_template.py` states and tests these: -- **Every committed prompt re-renders on its recorded version.** The prompt - sha256s committed in `docs/gpt61sol/` and `docs/haiku55/` (the seed digests - and the judge provenance records of releases 20260929 through 20261010) - re-render from each release's committed board on the version their verdicts - record, v1. This test is local only: it needs the audit grounding, which is - not committed. In CI, a property test checks that `prepare_audit` keeps any - judged case's prompt, verdict and sidecar bytes when the case is unchanged. +- **Every committed prompt re-renders on its recorded version.** Each prompt + sha256 committed in `docs/gpt61sol/` and `docs/haiku55/` (the seed digests + and the judge provenance records) re-renders on the version its verdict + records, v1. It renders from the committed board of the release that judged + it: release 20260929, 20260930 or 20261010. Release 20261006 judged + nothing. It carried release 20260930's audit while rewording nine reference + explanations, so those nine prompts do not re-render from its own board; + release 20261010 re-opened them. This test is local only: it needs the + audit grounding, which is not committed. In CI, property tests check that + `prepare_audit` keeps a judged case's files only on the exact bytes its + judge read. - **v2 differs from v1 only by the dropped clause.** The header diff is one deletion, `V1_REVIEW_CLAIM`. For any case, the two prompts differ only in their headers, and the v2 prompt is the v1 prompt without that clause. diff --git a/policybench/audit.py b/policybench/audit.py index 55ac34fb..4b801709 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -19,6 +19,7 @@ from __future__ import annotations +import hashlib import json import re import shutil @@ -343,14 +344,18 @@ def prepare_audit( cases//verdict.meta.json (its provenance sidecar) A case that already has a verdict (a seed a release driver copied in, or - an earlier run's) is rendered with the template version its sidecar - records (absent: v1). If that reproduces its prompt.md, the verdict - stands and the prompt keeps its bytes. Otherwise the case changed since it - was judged, or its version is unknown: the verdict and its sidecar are - dropped and the case is re-opened. New and re-opened cases are rendered - with ``template_version``, which the caller must name. The version a case - renders with comes from the sidecar or the caller, never from prompt.md: - a prompt that begins with another version's header is not adopted. + an earlier run's) keeps it only on the exact bytes its judge read + (:func:`_judged_prompt`): the case rendered on the template version its + sidecar records (absent: v1) must equal its prompt.md byte for byte, or, + when prompt.md is missing, hash to the ``prompt_sha256`` its sidecar + records. A kept case's files are left untouched. Otherwise the case + changed since it was judged, or its version or judged bytes are unknown: + the verdict and its sidecar are dropped and the case is re-opened. New + and re-opened cases are rendered with ``template_version``, which the + caller must name. The version a case renders with comes from the sidecar + or the caller, never from prompt.md: a prompt that begins with another + version's header is not adopted. Prompts are written as UTF-8 bytes, with + no newline translation. """ template_header(template_version) cases = build_audit_cases(country_dir, grounding_lookup=grounding_lookup) @@ -380,33 +385,55 @@ def prepare_audit( case_dir.mkdir(exist_ok=True) prompt_path = case_dir / "prompt.md" verdict_path = case_dir / "verdict.json" - new_prompt = None if verdict_path.exists(): - # A judged case renders with the template its verdict was - # judged on, so an unchanged case keeps its prompt's bytes. - judged_on = recorded_template_version(_sidecar(case_dir)) - if judged_on is not None: - seeded = render_case_prompt(case, template_version=judged_on) - # A verdict without its prompt keeps it, as before - # templates were versioned. - if not prompt_path.exists() or prompt_path.read_text() == seeded: - new_prompt = seeded - if new_prompt is None: - # Content-aware resumability: the case changed since it was - # classified (e.g. a model was re-run and now answers - # differently), or its template is unknown. Drop the stale - # verdict so the runner re-classifies it rather than - # reusing the old label. The provenance sidecar describes - # that verdict; a re-judge by the other runner must not - # inherit it. - verdict_path.unlink() - (case_dir / "verdict.meta.json").unlink(missing_ok=True) - if new_prompt is None: - new_prompt = render_case_prompt(case, template_version=template_version) - prompt_path.write_text(new_prompt) + judged = _judged_prompt(case, case_dir) + if judged is not None: + # The verdict stands on these bytes. A prompt.md the + # sidecar's hash vouched for is restored. + if not prompt_path.exists(): + prompt_path.write_bytes(judged) + continue + # Content-aware resumability: the case changed since it was + # classified (e.g. a model was re-run and now answers + # differently), or its template or judged bytes are unknown. + # Drop the stale verdict so the runner re-classifies it rather + # than reusing the old label. The provenance sidecar describes + # that verdict; a re-judge by the other runner must not + # inherit it. + verdict_path.unlink() + (case_dir / "verdict.meta.json").unlink(missing_ok=True) + prompt = render_case_prompt(case, template_version=template_version) + prompt_path.write_bytes(prompt.encode("utf-8")) return cases +def _judged_prompt(case: AuditCase, case_dir: Path) -> bytes | None: + """The prompt bytes a judged case's verdict stands on, or None when the + case must be re-opened. + + The case is rendered on the version its sidecar records (absent: v1; a + version it names no known one for gives None). Those bytes must be + prompt.md's exactly, and, when the sidecar records ``prompt_sha256`` (the + hash of the bytes its judge read), hash to it. Without prompt.md, that + hash is the only evidence of what the judge read, so a sidecar without + one gives None. + """ + meta = _sidecar(case_dir) + judged_on = recorded_template_version(meta) + if judged_on is None: + return None + rendered = render_case_prompt(case, template_version=judged_on).encode("utf-8") + recorded_sha256 = (meta or {}).get("prompt_sha256") + if recorded_sha256 is not None and ( + recorded_sha256 != hashlib.sha256(rendered).hexdigest() + ): + return None + prompt_path = case_dir / "prompt.md" + if prompt_path.exists(): + return rendered if prompt_path.read_bytes() == rendered else None + return rendered if recorded_sha256 is not None else None + + def _sidecar(case_dir: Path) -> dict | None: """A case's ``verdict.meta.json``, or None when it has none. diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index 35716028..fd7afa21 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -14,8 +14,11 @@ # (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex # reports in its log header, the model requested, the UTC timestamp, the # verdict's sha256, so a re-judged case can never keep the other runner's -# provenance, and the judge template version prompt.md begins with +# provenance, and the bytes Codex judged: a private copy of prompt.md, taken +# before it runs, whose sha256 the sidecar records as prompt_sha256 and whose +# judge template version it records as judge_template_version # (policybench.judge_template; null for a prompt audit-prepare did not render). +# A verdict is not published if prompt.md no longer holds those bytes. set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -75,6 +78,13 @@ classify_one() { # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar # left from a previous verdict describes a verdict that no longer exists. rm -f "$tmp" "$case_dir/verdict.meta.json" + # Codex reads a private copy, so the sidecar describes the bytes it judged + # even if audit-prepare rewrites prompt.md meanwhile. + judged=$(mktemp "${TMPDIR:-/tmp}/pb-codex-prompt.XXXXXX") || { + echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)"; return 0; } + cp "$prompt" "$judged" || { + rm -f "$judged"; echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)" + return 0; } codex exec \ --sandbox read-only \ --skip-git-repo-check \ @@ -84,9 +94,9 @@ classify_one() { -c model_reasoning_effort="$EFFORT" \ --output-schema "$SCHEMA" \ -o "$tmp" \ - - < "$prompt" > "$case_dir/codex.log" 2>&1 + - < "$judged" > "$case_dir/codex.log" 2>&1 if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ - "$case_dir/verdict.meta.json.tmp" "$prompt"; then + "$case_dir/verdict.meta.json.tmp" "$judged" "$prompt"; then mv -f "$tmp" "$out" mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" echo "[ok] $(basename "$case_dir")" @@ -94,21 +104,34 @@ classify_one() { rm -f "$tmp" "$case_dir/verdict.meta.json.tmp" echo "[FAIL] $(basename "$case_dir") (see codex.log)" fi + rm -f "$judged" } # Provenance sidecar for a validated verdict: the model Codex reports in its # log header (`model: ...`), the model requested (or "default"), the UTC time, # the verdict's sha256 so the sidecar cannot outlive the verdict it describes, -# and the judge template version of the prompt it was judged on. +# and the sha256 and judge template version of the copy Codex judged. Fails, +# so the verdict is not published, when prompt.md no longer holds that copy's +# bytes. write_provenance() { - verdict_path="$1"; log_path="$2"; meta_out="$3"; prompt_path="$4" + verdict_path="$1"; log_path="$2"; meta_out="$3"; judged_path="$4" + prompt_path="$5" "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" \ - "$prompt_path" "$SCRIPT_DIR" <<'PY' + "$judged_path" "$prompt_path" "$SCRIPT_DIR" <<'PY' import datetime, hashlib, json, re, sys from pathlib import Path -verdict_path, log_path, meta_out, requested, prompt_path, script_dir = sys.argv[1:7] +(verdict_path, log_path, meta_out, requested, judged_path, prompt_path, + script_dir) = sys.argv[1:8] sys.path.insert(0, str(Path(script_dir).resolve().parent)) from policybench.judge_template import template_version_of +judged = open(judged_path, "rb").read() +try: + live = open(prompt_path, "rb").read() +except OSError: + live = None +if live != judged: + print("prompt.md changed while Codex judged it", file=sys.stderr) + sys.exit(1) verdict = open(verdict_path, "rb").read() reported = [] try: @@ -123,7 +146,9 @@ meta = { "judge_model_requested": requested, "judge_model_reported": reported, "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "judge_template_version": template_version_of(open(prompt_path, "rb").read()), + # The exact bytes Codex read on stdin, copied before it ran. + "prompt_sha256": hashlib.sha256(judged).hexdigest(), + "judge_template_version": template_version_of(judged), } json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True) PY diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 5717c79c..22bcd973 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -295,6 +295,11 @@ def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): assert meta["judge_model_reported"] == ["gpt-5.6-sol"] # The fixture's prompt is no judge template's. assert meta["judge_template_version"] is None + # The bytes Codex judged, from its private copy of prompt.md. + assert ( + meta["prompt_sha256"] + == hashlib.sha256((case_dir / "prompt.md").read_bytes()).hexdigest() + ) assert ( meta["verdict_sha256"] == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() @@ -538,3 +543,43 @@ def test_both_runners_record_the_judge_template_version(tmp_path: Path): assert meta["judge_runner"] == f"scripts/{runner}" assert meta["judge_template_version"] == version assert template_version_problems(audit_dir) == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_codex_publishes_no_verdict_for_a_prompt_rewritten_while_it_judged( + tmp_path: Path, +): + """Codex judges a v1 prompt; audit-prepare rewrites prompt.md to v2 while + it runs. The verdict describes bytes prompt.md no longer holds, so the + runner publishes neither it nor a sidecar, and leaves no copy behind.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + v1 = template_header(1) + "\nCOUNTRY: US\nClassify this miss.\n" + v2 = template_header(2) + "\nCOUNTRY: US\nClassify this miss.\n" + (case_dir / "prompt.md").write_text(v1) + rewrite = tmp_path / "v2.md" + rewrite.write_text(v2) + verdict = tmp_path / "canned_verdict.json" + copies = tmp_path / "copies" + copies.mkdir() + _fake_cli( + tmp_path / "bin" / "codex", + 'out=""; while [ $# -gt 0 ]; do' + ' if [ "$1" = -o ]; then out="$2"; shift; fi; shift; done\n' + "cat >/dev/null\n" + 'echo "model: gpt-5.6-sol"\n' + f'cp "{rewrite}" "{case_dir / "prompt.md"}"\n' + f'cat "{verdict}" > "$out"\n', + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "TMPDIR": str(copies)}, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + assert f"[FAIL] {case_dir.name}" in result.stdout + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + assert list(copies.iterdir()) == [] + assert (case_dir / "prompt.md").read_text() == v2 diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py index 8c757d6e..0de652e3 100644 --- a/tests/test_judge_template.py +++ b/tests/test_judge_template.py @@ -2,12 +2,13 @@ the version each was judged on. Invariants: -- every committed prompt re-renders on its recorded version: the prompt - sha256s that docs/gpt61sol/ and docs/haiku55/ commit re-render from each - release's committed board on the version their verdicts record (local only, - for it needs the audit grounding), and prepare_audit keeps a judged case's - prompt, verdict and sidecar bytes when the case re-renders to them with its - recorded version (absent: v1); +- every committed prompt re-renders on its recorded version: each prompt + sha256 that docs/gpt61sol/ and docs/haiku55/ commit re-renders, on the + version its verdict records, from the committed board of the release that + judged it (local only, for it needs the audit grounding); and prepare_audit + keeps a judged case's files only on the exact bytes its judge read: the + case rendered on its recorded version (absent: v1), equal to prompt.md and + to the sidecar's prompt_sha256 when it records one; - v2 differs from v1 only by the dropped clause: the header diff is one deletion, V1_REVIEW_CLAIM, and the rest of a prompt does not depend on the version; @@ -180,7 +181,7 @@ def test_recorded_version_is_the_field_when_it_names_one(value): # --- Rendering ------------------------------------------------------------------- TEXT = st.text( - alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\r"), + alphabet=st.characters(blacklist_categories=("Cs",)), max_size=60, ) @@ -552,19 +553,122 @@ def meta(scenario: str, **fields) -> None: with pytest.raises(SystemExit, match="5 verdicts disagree with their prompt.md"): _collect_cli(board, audit, tmp_path / "out") assert not (tmp_path / "out").exists() - # Preparing again re-opens every flagged case except the verdict that - # lost its prompt, which keeps its verdict and gets its v1 prompt back. + # Preparing again re-opens every flagged case. The verdict that lost its + # prompt has no sidecar, so nothing records the bytes its judge read. prepare_audit(board, audit, template_version=2) assert template_version_problems(audit) == [] - assert (case_dir["s0"] / "verdict.json").is_file() - assert (case_dir["s0"] / "prompt.md").read_text() == render_case_prompt( - cases["s0"], template_version=1 - ) - for scenario in ("s1", "s2", "s3", "s4"): + for scenario in ("s0", "s1", "s2", "s3", "s4"): assert not (case_dir[scenario] / "verdict.json").exists(), scenario - assert (case_dir[scenario] / "prompt.md").read_text() == render_case_prompt( + assert (case_dir[scenario] / "prompt.md").read_bytes() == render_case_prompt( cases[scenario], template_version=2 - ) + ).encode("utf-8") + + +def _judged_case(tmp_path: Path, answer: float = 250.0, **sidecar): + """A one-case board whose case is judged on v1, with ``sidecar``'s fields + in its sidecar. Returns the board, the audit, the case and its dir.""" + board = _board(tmp_path / "us", {"s0": answer}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, **sidecar}) + return board, audit, case, case_dir + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def test_a_verdict_without_its_prompt_stands_only_on_its_recorded_hash(tmp_path): + """Without prompt.md, the sidecar's prompt_sha256 is the only record of + what the judge read. A verdict whose hash is the case rendered on its + version keeps its files, and its prompt comes back byte for byte; one + with no hash, or another hash, is re-opened.""" + for name, recorded in ( + ("matching", "v1"), + ("no hash", None), + ("another hash", "v2"), + ): + root = tmp_path / name.replace(" ", "-") + board = _board(root / "us", {"s0": 250.0}) + audit = root / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + renders = { + v: render_case_prompt(case, template_version=v).encode("utf-8") + for v in JUDGE_TEMPLATE_HEADERS + } + fields = {TEMPLATE_VERSION_FIELD: 1} + if recorded is not None: + fields["prompt_sha256"] = _sha256(renders[int(recorded[1])]) + _judge(case_dir, fields) + verdict = (case_dir / "verdict.json").read_bytes() + (case_dir / "prompt.md").unlink() + assert dict(template_version_problems(audit)) == { + case.case_id: "a verdict without prompt.md" + } + prepare_audit(board, audit, template_version=2) + if recorded == "v1": + assert (case_dir / "verdict.json").read_bytes() == verdict, name + assert (case_dir / "prompt.md").read_bytes() == renders[1], name + else: + assert not (case_dir / "verdict.json").exists(), name + assert not (case_dir / "verdict.meta.json").exists(), name + assert (case_dir / "prompt.md").read_bytes() == renders[2], name + assert template_version_problems(audit) == [] + + +def test_a_changed_case_without_its_prompt_is_reopened(tmp_path): + """The review's case: a verdict judged on m1's $250 answer loses its + prompt.md, and m1 is re-run and answers $999. The case renders anew, so + the verdict is dropped, even though its sidecar hashes the old prompt.""" + board, audit, case, case_dir = _judged_case(tmp_path) + old = (case_dir / "prompt.md").read_bytes() + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + meta["prompt_sha256"] = _sha256(old) + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + (case_dir / "prompt.md").unlink() + _board(board, {"s0": 999.0}) + (changed,) = prepare_audit(board, audit, template_version=1) + assert changed.case_id == case.case_id + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + new = (case_dir / "prompt.md").read_bytes() + assert new != old and b"999" in new + + +def test_a_sidecar_hash_that_is_not_the_prompt_reopens_the_case(tmp_path): + """A prompt.md that renders right but is not the bytes the sidecar says + its judge read is not the verdict's prompt.""" + board, audit, case, case_dir = _judged_case(tmp_path, prompt_sha256="0" * 64) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + + +def test_prompts_compare_and_keep_exact_bytes(tmp_path): + """Byte equality, with no newline translation either way: a case whose + text holds a carriage return keeps its verdict and its prompt's bytes on + every preparation; a prompt.md with CRLF line endings is not the case's + rendering, so its verdict is re-opened and the prompt rewritten exactly.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + predictions = pd.read_csv(board / "predictions.csv") + predictions["explanation"] = "Gross income\rtest only." + predictions.to_csv(board / "predictions.csv", index=False) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + rendered = render_case_prompt(case, template_version=1).encode("utf-8") + assert b"Gross income\rtest only." in rendered + case_dir = audit / "cases" / case.case_id + assert (case_dir / "prompt.md").read_bytes() == rendered + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, "prompt_sha256": _sha256(rendered)}) + before = _tree(audit) + for version in sorted(JUDGE_TEMPLATE_HEADERS): + prepare_audit(board, audit, template_version=version) + assert _tree(audit) == before, version + (case_dir / "prompt.md").write_bytes(rendered.replace(b"\n", b"\r\n")) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "prompt.md").read_bytes() == rendered def _collect_cli(board: Path, audit: Path, out: Path) -> None: @@ -682,6 +786,48 @@ def test_every_renderer_call_names_its_version(): assert found.get(relative) == counts, relative +# Where Python code may read a version off a prompt: only to check a tree. +# (The runners' embedded Python also records what a judge read.) +TEMPLATE_VERSION_OF_CALLERS = {("policybench/audit.py", "template_version_problems")} + + +def test_only_the_tree_check_reads_a_version_off_a_prompt(): + """Static, closing the scan's gap for a version read into a variable and + passed on: template_version_of is called nowhere in policybench/, + scripts/ or reference_audit/ but in template_version_problems, which + renders nothing.""" + callers = set() + for part in ("policybench", "scripts", "reference_audit"): + for path in sorted((ROOT / part).rglob("*.py")): + relative = path.relative_to(ROOT).as_posix() + tree = ast.parse(path.read_text(), filename=relative) + for function in ast.walk(tree): + if not isinstance(function, ast.FunctionDef | ast.AsyncFunctionDef): + continue + for call in ast.walk(function): + if ( + isinstance(call, ast.Call) + and _called_name(call) == "template_version_of" + and function.name != "template_version_of" + ): + callers.add((relative, function.name)) + module_level = [ + node + for node in tree.body + if not isinstance( + node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef + ) + ] + for node in module_level: + for call in ast.walk(node): + if ( + isinstance(call, ast.Call) + and _called_name(call) == "template_version_of" + ): + callers.add((relative, "")) + assert callers == TEMPLATE_VERSION_OF_CALLERS + + # The release drivers of releases judged before versions existed: each # judged every verdict on v1. V1_DRIVERS = ( @@ -718,55 +864,91 @@ def test_each_past_release_driver_renders_on_v1_by_name(): # A judged case's prompt.md, as the property draws it. -PROMPT_KINDS = ("v1", "v2", "v1 + text", "v2 + text", "no header") +PROMPT_KINDS = ( + "v1", + "v2", + "v1 + text", + "v2 + text", + "v1 CRLF", + "v2 CRLF", + "no header", + "missing", +) JUDGED_STATES = sorted(state for state in SEED_STATES if state != "unjudged") +# What the sidecar's prompt_sha256 hashes: nothing (no field), each version's +# rendering of the case, or bytes no prompt has. +HASHES = (None, "v1", "v2", "other") @PROPERTY @given( st.sampled_from(JUDGED_STATES), st.sampled_from(PROMPT_KINDS), + st.sampled_from(HASHES), st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), TEXT, ) -def test_a_verdict_is_kept_on_its_recorded_version_alone(state, kind, version, text): - """Never inferred: prepare_audit keeps an unchanged judged case's verdict - exactly when prompt.md is the case rendered on the version its sidecar - records. A prompt.md on another version is re-opened, though - template_version_of reads a version off it and though the caller may name - that very version; a re-opened case renders on the caller's version.""" +def test_a_verdict_is_kept_on_its_recorded_version_alone( + state, kind, hashed, version, text +): + """Never inferred, and byte for byte: prepare_audit keeps an unchanged + judged case's verdict exactly when the case rendered on the version its + sidecar records is the bytes its judge read. Those are prompt.md's bytes, + which must also hash to the sidecar's prompt_sha256 when it records one; + without prompt.md, that hash alone. A prompt.md on another version is + re-opened, though template_version_of reads a version off it and though + the caller may name that very version, and so is a CRLF copy. A kept + case's files are untouched; a re-opened case renders on the caller's + version.""" with tempfile.TemporaryDirectory() as scratch: root = Path(scratch) board = _board(root / "us", {"s0": 250.0}) audit = root / "audit" (case,) = prepare_audit(board, audit, template_version=version) case_dir = audit / "cases" / case.case_id - if kind == "no header": - prompt = f"Classify this miss.\n{text}" + renders = { + v: render_case_prompt(case, template_version=v).encode("utf-8") + for v in JUDGE_TEMPLATE_HEADERS + } + if kind == "missing": + prompt = None + elif kind == "no header": + prompt = f"Classify this miss.\n{text}".encode() else: - prompt = render_case_prompt(case, template_version=int(kind[1])) + prompt = renders[int(kind[1])] if kind.endswith("+ text"): - prompt += f"\n{text}." - (case_dir / "prompt.md").write_text(prompt) - _judge(case_dir, SEED_STATES[state]) + prompt += f"\n{text}.".encode() + if kind.endswith("CRLF"): + prompt = prompt.replace(b"\n", b"\r\n") + if prompt is None: + (case_dir / "prompt.md").unlink() + else: + (case_dir / "prompt.md").write_bytes(prompt) + sidecar = SEED_STATES[state] + recorded_sha256 = None + if sidecar is not None and hashed is not None: + recorded_sha256 = ( + "0" * 64 if hashed == "other" else _sha256(renders[int(hashed[1])]) + ) + sidecar = {**sidecar, "prompt_sha256": recorded_sha256} + _judge(case_dir, sidecar) before = _tree(audit) - recorded = recorded_template_version(SEED_STATES[state]) - kept = recorded is not None and prompt == render_case_prompt( - case, template_version=recorded + recorded = recorded_template_version(sidecar) + judged = renders.get(recorded) + kept = ( + judged is not None + and recorded_sha256 in (None, _sha256(judged)) + and (prompt == judged if prompt is not None else bool(recorded_sha256)) ) prepare_audit(board, audit, template_version=version) after = _tree(audit) - names = ("prompt.md", "verdict.json", "verdict.meta.json") if kept: - for name in names: - key = f"cases/{case.case_id}/{name}" - assert after.get(key) == before.get(key), (state, kind, name) + restored = {f"cases/{case.case_id}/prompt.md": judged} + assert after == {**before, **restored}, (state, kind, hashed) else: - assert not (case_dir / "verdict.json").exists(), (state, kind) - assert not (case_dir / "verdict.meta.json").exists(), (state, kind) - assert (case_dir / "prompt.md").read_text() == render_case_prompt( - case, template_version=version - ) + assert not (case_dir / "verdict.json").exists(), (state, kind, hashed) + assert not (case_dir / "verdict.meta.json").exists() + assert (case_dir / "prompt.md").read_bytes() == renders[version] assert template_version_problems(audit) == [] @@ -907,7 +1089,10 @@ def test_every_committed_prompt_rerenders_on_its_recorded_version(tmp_path, rele """Local only: release ``release``'s board, read from the commit that committed it, renders every judged case of its audit, on the version its committed record names, to the prompt sha256 that record commits; and it - renders no other case.""" + renders no other case. Release 20261006 judged nothing: it carried + release 20260930's audit, which test_the_committed_prompt_records_agree + checks, while rewording nine explanations that release 20261010 + re-opened, so its own board is not one an audit was rendered from.""" if not GROUNDING.is_file(): pytest.skip("the audit grounding is not on this machine") assert hashlib.sha256(GROUNDING.read_bytes()).hexdigest() == GROUNDING_SHA256 From daf038bcd14c2616f4500109ba66cf99ebdeb3b4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 07:03:05 -0400 Subject: [PATCH 4/8] Reword _judged_prompt's docstring Co-Authored-By: Claude Opus 5.5 --- policybench/audit.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/policybench/audit.py b/policybench/audit.py index 4b801709..87087ccf 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -411,12 +411,11 @@ def _judged_prompt(case: AuditCase, case_dir: Path) -> bytes | None: """The prompt bytes a judged case's verdict stands on, or None when the case must be re-opened. - The case is rendered on the version its sidecar records (absent: v1; a - version it names no known one for gives None). Those bytes must be - prompt.md's exactly, and, when the sidecar records ``prompt_sha256`` (the - hash of the bytes its judge read), hash to it. Without prompt.md, that - hash is the only evidence of what the judge read, so a sidecar without - one gives None. + The case is rendered on the version its sidecar records (absent: v1; an + unknown version gives None). Those bytes must be prompt.md's exactly, + and, when the sidecar records ``prompt_sha256`` (the hash of the bytes + its judge read), hash to it. Without prompt.md, that hash is the only + evidence of what the judge read, so a sidecar without one gives None. """ meta = _sidecar(case_dir) judged_on = recorded_template_version(meta) From 0770d997a7deff9e8ea53e052cd21045250e6620 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 07:35:21 -0400 Subject: [PATCH 5/8] Refuse verdicts on rewritten bytes and publish each sidecar first Answers review round 2 of #210: - audit-collect refuses a verdict whose sidecar's prompt_sha256 is not prompt.md's bytes (prompt_hash_problems), such as one published after an audit-prepare that raced its runner rewrote the case. - Both runners publish verdict.meta.json before verdict.json, so an interrupted publish leaves at most a sidecar without a verdict, never a verdict that audit-prepare would read as legacy v1 with no hash. - run_audit_codex.sh traps EXIT, TERM and INT in each worker to remove its private prompt copy. - Tests: the race at collect time, an mv shim that kills each runner's worker between its two moves, a Codex worker terminated mid-judge, and no hash problems after any preparation in both properties. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 10 ++++- policybench/audit.py | 41 +++++++++++++++++++- policybench/cli.py | 13 ++++--- scripts/run_audit_claude.sh | 6 ++- scripts/run_audit_codex.sh | 11 +++++- tests/test_judge_provenance.py | 71 ++++++++++++++++++++++++++++++++++ tests/test_judge_template.py | 33 ++++++++++++++++ 7 files changed, 174 insertions(+), 11 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index f0ab7d4d..f39957b3 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -75,7 +75,10 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` that copy's sha256 (`prompt_sha256`) and version (`judge_template_version`) in `verdict.meta.json`. A verdict is not published if `prompt.md` no longer holds those bytes. A prompt that begins with no version's header, one - `audit-prepare` did not write, records `null`. + `audit-prepare` did not write, records `null`. A runner publishes the + sidecar before the verdict. An interrupted publish can leave a sidecar + without a verdict, which the next run removes, but never a new verdict + without its sidecar. - A sidecar without the field, or a verdict without a sidecar, was judged on v1 (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars written before versions existed, not a guess from the prompt. @@ -99,7 +102,10 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` kept case's prompt keeps the seed's bytes. - A tree may mix versions. `audit-collect` refuses a tree in which a verdict records an unknown version, or one other than the version its `prompt.md` - begins with (`policybench.audit.template_version_problems`). Re-running + begins with (`policybench.audit.template_version_problems`). It also + refuses a verdict whose sidecar's `prompt_sha256` is not `prompt.md`'s + bytes (`prompt_hash_problems`), such as a verdict published after an + `audit-prepare` that raced its runner rewrote the case. Re-running `audit-prepare` re-opens those cases. - A version's text never changes once a verdict is judged on it; changing the template means adding a version. `tests/test_judge_template.py` pins each diff --git a/policybench/audit.py b/policybench/audit.py index 87087ccf..e56ca013 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -486,6 +486,39 @@ def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: return problems +def prompt_hash_problems(audit_dir: Path) -> list[tuple[str, str]]: + """Each judged case whose prompt.md is not the bytes its judge read. + + A runner records the sha256 of the bytes its judge read as the sidecar's + ``prompt_sha256`` (sidecars from before the runners did record none, and + are not checked). A verdict published after prompt.md was rewritten, say + by an audit-prepare racing the runner, keeps the hash of the old bytes, + so it disagrees. A verdict without prompt.md is + :func:`template_version_problems`' to report. Returns ``(case_id, + problem)`` for each case that fails, in case order. + """ + problems: list[tuple[str, str]] = [] + cases_root = audit_dir / "cases" + if not cases_root.is_dir(): + return problems + for case_dir in sorted(cases_root.iterdir()): + prompt_path = case_dir / "prompt.md" + if not ((case_dir / "verdict.json").is_file() and prompt_path.is_file()): + continue + recorded = (_sidecar(case_dir) or {}).get("prompt_sha256") + if recorded is None: + continue + if hashlib.sha256(prompt_path.read_bytes()).hexdigest() != recorded: + problems.append( + ( + case_dir.name, + "prompt.md is not the bytes its judge read (its sidecar's " + "prompt_sha256)", + ) + ) + return problems + + # --- Hedge detection ----------------------------------------------------------- # Published diagnoses must be definitive. These patterns catch the judge @@ -580,7 +613,7 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] """Fold verdicts into the annotation schema. Returns ``{"row": ..., "case": ..., "missing": ..., "hedged": ..., - "template": ...}``. + "template": ..., "prompt_hash": ...}``. ``row`` and ``case`` match the committed annotation CSV columns, extended with ``rationale`` and ``reference_suspect`` so the classifier's reasoning is preserved. ``missing`` lists cases whose verdict has not yet been @@ -589,7 +622,8 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] delete those ``verdict.json`` files and re-run the classifier rather than shipping them. ``template`` lists the judged cases whose prompt and verdict disagree on the judge template version - (:func:`template_version_problems`). + (:func:`template_version_problems`), and ``prompt_hash`` those whose + prompt.md is not the bytes their judge read (:func:`prompt_hash_problems`). """ manifest = _load_manifest(audit_dir) cases_root = audit_dir / "cases" @@ -738,4 +772,7 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] "template": pd.DataFrame( template_version_problems(audit_dir), columns=["case_id", "problem"] ), + "prompt_hash": pd.DataFrame( + prompt_hash_problems(audit_dir), columns=["case_id", "problem"] + ), } diff --git a/policybench/cli.py b/policybench/cli.py index cc474617..3b9376b7 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -1407,16 +1407,19 @@ def main(): country_dir = Path(args.country_dir) out = collect_audit(country_dir, Path(args.audit_dir)) - template = out["template"] - if not template.empty: + template, prompt_hash = out["template"], out["prompt_hash"] + if not (template.empty and prompt_hash.empty): # A verdict judged on other words than its prompt.md's is not # this case's verdict. for case_id, problem in template.itertuples(index=False): print(f"TEMPLATE: {case_id}: {problem}") + for case_id, problem in prompt_hash.itertuples(index=False): + print(f"PROMPT: {case_id}: {problem}") + disagree = {*template["case_id"], *prompt_hash["case_id"]} raise SystemExit( - f"{len(template)} verdicts disagree with their prompt.md on the " - "judge template version; re-run audit-prepare, which re-opens " - "them, and re-judge" + f"{len(disagree)} verdicts disagree with their prompt.md on the " + "judge template version or the bytes their judge read; re-run " + "audit-prepare, which re-opens them, and re-judge" ) hedged = out["hedged"] if not hedged.empty and not args.allow_hedged: diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 31f4190f..2f33388e 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -691,8 +691,12 @@ classify_one() { rm -f "$judged" if [ "$extracted" = 0 ] && verdict_ok "$tmp"; then if problem=$(case_ok "$tmp" "$name" 2>&1); then - mv -f "$tmp" "$out" + # The sidecar first: an interrupted publish leaves at most a sidecar + # without a verdict, which the next run removes, and never a verdict + # without its sidecar, which audit-prepare would read as a legacy v1 + # verdict with no record of the bytes its judge read. mv -f "$meta_tmp" "$case_dir/verdict.meta.json" + mv -f "$tmp" "$out" echo "[ok] $name" else # Answered, but not for this case's models: pending, and the run goes on. diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index fd7afa21..276eaa51 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -82,6 +82,11 @@ classify_one() { # even if audit-prepare rewrites prompt.md meanwhile. judged=$(mktemp "${TMPDIR:-/tmp}/pb-codex-prompt.XXXXXX") || { echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)"; return 0; } + # Each case runs in its own background subshell: a worker terminated + # mid-case still removes its copy. + trap 'rm -f "$judged"' EXIT + trap 'rm -f "$judged"; exit 143' TERM + trap 'rm -f "$judged"; exit 130' INT cp "$prompt" "$judged" || { rm -f "$judged"; echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)" return 0; } @@ -97,8 +102,12 @@ classify_one() { - < "$judged" > "$case_dir/codex.log" 2>&1 if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ "$case_dir/verdict.meta.json.tmp" "$judged" "$prompt"; then - mv -f "$tmp" "$out" + # The sidecar first: an interrupted publish leaves at most a sidecar + # without a verdict, which the next run removes, and never a verdict + # without its sidecar, which audit-prepare would read as a legacy v1 + # verdict with no record of the bytes its judge read. mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" + mv -f "$tmp" "$out" echo "[ok] $(basename "$case_dir")" else rm -f "$tmp" "$case_dir/verdict.meta.json.tmp" diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 22bcd973..8a40f19e 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -23,6 +23,7 @@ from policybench.audit import ( # noqa: E402 AUDIT_OUTPUT_SCHEMA, + prompt_hash_problems, template_version_problems, ) from policybench.judge_template import template_header # noqa: E402 @@ -583,3 +584,73 @@ def test_codex_publishes_no_verdict_for_a_prompt_rewritten_while_it_judged( assert not (case_dir / "verdict.meta.json").exists() assert list(copies.iterdir()) == [] assert (case_dir / "prompt.md").read_text() == v2 + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +@pytest.mark.parametrize("runner", ["run_audit_claude.sh", "run_audit_codex.sh"]) +def test_an_interrupted_publish_never_leaves_a_verdict_without_its_sidecar( + tmp_path: Path, runner +): + """A worker killed between publishing its two files leaves at most a + sidecar without a verdict, never a verdict that audit-prepare would read + as a legacy v1 verdict with no record of the bytes its judge read. Here + the first move succeeds and the worker is then terminated. The next run + judges the case again and publishes both.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + (case_dir / "prompt.md").write_text( + template_header(1) + "\nCOUNTRY: US\nClassify this miss.\n" + ) + shim = tmp_path / "shim" + shim.mkdir() + _fake_cli(shim / "mv", '/bin/mv "$@" || exit $?\nkill -TERM "$PPID"\n') + interrupted = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "PATH": f"{shim}:{env['PATH']}"}, + cwd=tmp_path, + ) + assert f"[ok] {case_dir.name}" not in interrupted.stdout, interrupted.stdout + assert not (case_dir / "verdict.json").exists() + assert template_version_problems(audit_dir) == [] + result = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == f"scripts/{runner}" + assert meta["judge_template_version"] == 1 + assert ( + meta["prompt_sha256"] + == hashlib.sha256((case_dir / "prompt.md").read_bytes()).hexdigest() + ) + assert template_version_problems(audit_dir) == [] + assert prompt_hash_problems(audit_dir) == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_a_terminated_codex_worker_removes_its_prompt_copy(tmp_path: Path): + """The worker judging a case is terminated while Codex runs: it removes + its private copy of prompt.md and publishes nothing.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + copies = tmp_path / "copies" + copies.mkdir() + _fake_cli( + tmp_path / "bin" / "codex", + 'cat >/dev/null\necho "model: gpt-5.6-sol"\nkill -TERM "$PPID"\n', + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "TMPDIR": str(copies)}, + cwd=tmp_path, + ) + assert "audit complete: 0/1" in result.stdout, result.stdout + result.stderr + assert list(copies.iterdir()) == [] + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py index 0de652e3..811630f8 100644 --- a/tests/test_judge_template.py +++ b/tests/test_judge_template.py @@ -49,6 +49,7 @@ build_audit_cases, collect_audit, prepare_audit, + prompt_hash_problems, render_case_prompt, template_version_problems, ) @@ -404,6 +405,7 @@ def test_prepare_keeps_each_judged_case_on_its_own_template(plan, version): cases[scenario], template_version=version ).encode("utf-8") assert template_version_problems(audit) == [] + assert prompt_hash_problems(audit) == [] prepare_audit(board, audit, template_version=version) assert _tree(audit) == after @@ -645,6 +647,36 @@ def test_a_sidecar_hash_that_is_not_the_prompt_reopens_the_case(tmp_path): assert not (case_dir / "verdict.json").exists() +def test_collect_refuses_a_verdict_published_on_rewritten_bytes(tmp_path): + """The review's race: a runner's judge reads the $250 case on v1; an + audit-prepare racing it re-runs m1 at $999 and rewrites prompt.md, on the + same version; then the runner publishes. The verdict's sidecar truthfully + hashes the $250 prompt, so audit-collect refuses it, and audit-prepare + re-opens it.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + rewritten = (case_dir / "prompt.md").read_bytes() + assert rewritten != judged + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, "prompt_sha256": _sha256(judged)}) + assert template_version_problems(audit) == [] + problem = "prompt.md is not the bytes its judge read (its sidecar's prompt_sha256)" + assert prompt_hash_problems(audit) == [(case.case_id, problem)] + out = collect_audit(board, audit) + assert list(out["prompt_hash"].itertuples(index=False)) == [(case.case_id, problem)] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "prompt.md").read_bytes() == rewritten + assert prompt_hash_problems(audit) == [] + + def test_prompts_compare_and_keep_exact_bytes(tmp_path): """Byte equality, with no newline translation either way: a case whose text holds a carriage return keeps its verdict and its prompt's bytes on @@ -950,6 +982,7 @@ def test_a_verdict_is_kept_on_its_recorded_version_alone( assert not (case_dir / "verdict.meta.json").exists() assert (case_dir / "prompt.md").read_bytes() == renders[version] assert template_version_problems(audit) == [] + assert prompt_hash_problems(audit) == [] # --- Every committed prompt re-renders on its recorded version ------------------ From 18595d71d5a1ae9e5360c365110f68a5392e2e90 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:15:01 -0400 Subject: [PATCH 6/8] Never remove a sidecar for a verdict prepare_audit did not drop Answers review round 3 of #210: an audit-prepare that dropped a stale or truncated verdict also removed the sidecar a runner had just published, stranding the runner's verdict without its record. - prepare_audit removes a sidecar only together with the verdict it describes (verdict_sha256), or a legacy sidecar that records none, and removes verdict.json only while it holds the bytes it read. - A verdict is kept only when its sidecar, if it records verdict_sha256, describes it. - Both runners remove an invalid verdict.json when they start a case, so no verdict sits beside a new sidecar before the run's verdict is published. - Tests: the review's schedule step by step and end to end on the real Codex runner (an mv shim runs audit-prepare between the two moves), the drop rule, the sidecar-describes-verdict keep rule, a property dimension for it, and interruption tests that start from a truncated verdict. - Docs state what holds when audit-prepare and a runner share a tree. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 30 +++++++---- policybench/audit.py | 76 ++++++++++++++++++++------- scripts/run_audit_claude.sh | 7 ++- scripts/run_audit_codex.sh | 7 ++- tests/test_judge_provenance.py | 82 +++++++++++++++++++++++++++-- tests/test_judge_template.py | 95 +++++++++++++++++++++++++++++++--- 6 files changed, 255 insertions(+), 42 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index f39957b3..504e3c6f 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -75,10 +75,11 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` that copy's sha256 (`prompt_sha256`) and version (`judge_template_version`) in `verdict.meta.json`. A verdict is not published if `prompt.md` no longer holds those bytes. A prompt that begins with no version's header, one - `audit-prepare` did not write, records `null`. A runner publishes the - sidecar before the verdict. An interrupted publish can leave a sidecar - without a verdict, which the next run removes, but never a new verdict - without its sidecar. + `audit-prepare` did not write, records `null`. Before it judges a case, a + runner removes the case's invalid `verdict.json` and stale sidecar, and it + publishes the sidecar before the verdict. An interrupted publish can leave + a sidecar without a verdict, which the next run removes, but never a new + verdict without its sidecar. - A sidecar without the field, or a verdict without a sidecar, was judged on v1 (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars written before versions existed, not a guess from the prompt. @@ -87,11 +88,22 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` `prompt.md` byte for byte and, when the sidecar records `prompt_sha256` (the hash of the bytes its judge read), matches it. Without `prompt.md`, the verdict stands only if that hash matches, and the prompt is restored. - A kept case's files are not rewritten. Otherwise the case changed since it - was judged, or its version or judged bytes are unknown: the verdict and - its sidecar are dropped and the case is re-opened. A `prompt.md` on - another version is re-opened, not adopted. New and re-opened cases use the - version the caller names. + A sidecar that records `verdict_sha256` must also describe the verdict. A + kept case's files are not rewritten. Otherwise the case changed since it + was judged, or its version or judged bytes are unknown, and the case is + re-opened: the verdict is dropped, and so is its sidecar when the sidecar + describes that verdict or records no `verdict_sha256`. A sidecar that + describes another verdict stays, such as the record of a verdict a runner + is still publishing. A `prompt.md` on another version is re-opened, not + adopted. New and re-opened cases use the version the caller names. +- `audit-prepare` may run while a runner judges the same tree, with one + runner per case and one `audit-prepare` at a time. Once both finish, no + verdict passes `audit-collect`, or survives the next `audit-prepare`, on + bytes its judge did not read. That holds because a runner never leaves a + new verdict without its sidecar, `audit-prepare` never removes another + verdict's sidecar, and `audit-collect` checks each recorded hash. A + verdict judged before a racing `audit-prepare` rewrote its case is + refused, not kept. Run `audit-collect` after both finish. - Each release driver names the version its new and re-opened cases render on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, diff --git a/policybench/audit.py b/policybench/audit.py index e56ca013..0e60044f 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -385,8 +385,9 @@ def prepare_audit( case_dir.mkdir(exist_ok=True) prompt_path = case_dir / "prompt.md" verdict_path = case_dir / "verdict.json" - if verdict_path.exists(): - judged = _judged_prompt(case, case_dir) + verdict = _read_or_none(verdict_path) + if verdict is not None: + judged = _judged_prompt(case, case_dir, verdict) if judged is not None: # The verdict stands on these bytes. A prompt.md the # sidecar's hash vouched for is restored. @@ -397,27 +398,41 @@ def prepare_audit( # classified (e.g. a model was re-run and now answers # differently), or its template or judged bytes are unknown. # Drop the stale verdict so the runner re-classifies it rather - # than reusing the old label. The provenance sidecar describes - # that verdict; a re-judge by the other runner must not - # inherit it. - verdict_path.unlink() - (case_dir / "verdict.meta.json").unlink(missing_ok=True) + # than reusing the old label. + _drop_verdict(case_dir, verdict) prompt = render_case_prompt(case, template_version=template_version) prompt_path.write_bytes(prompt.encode("utf-8")) return cases -def _judged_prompt(case: AuditCase, case_dir: Path) -> bytes | None: - """The prompt bytes a judged case's verdict stands on, or None when the - case must be re-opened. +def _read_or_none(path: Path) -> bytes | None: + """A file's bytes, or None when it does not exist (a runner may remove a + file between a check and a read).""" + try: + return path.read_bytes() + except FileNotFoundError: + return None + - The case is rendered on the version its sidecar records (absent: v1; an - unknown version gives None). Those bytes must be prompt.md's exactly, - and, when the sidecar records ``prompt_sha256`` (the hash of the bytes - its judge read), hash to it. Without prompt.md, that hash is the only - evidence of what the judge read, so a sidecar without one gives None. +def _judged_prompt(case: AuditCase, case_dir: Path, verdict: bytes) -> bytes | None: + """The prompt bytes the judged case's ``verdict`` stands on, or None when + the case must be re-opened. + + A sidecar that records ``verdict_sha256`` must describe ``verdict``: + otherwise it is another verdict's record, such as one a runner has + published and whose verdict has not arrived yet. The case is rendered on + the version the sidecar records (absent: v1; an unknown version gives + None). Those bytes must be prompt.md's exactly, and, when the sidecar + records ``prompt_sha256`` (the hash of the bytes its judge read), hash to + it. Without prompt.md, that hash is the only evidence of what the judge + read, so a sidecar without one gives None. """ meta = _sidecar(case_dir) + recorded_verdict = (meta or {}).get("verdict_sha256") + if recorded_verdict is not None and ( + recorded_verdict != hashlib.sha256(verdict).hexdigest() + ): + return None judged_on = recorded_template_version(meta) if judged_on is None: return None @@ -433,16 +448,41 @@ def _judged_prompt(case: AuditCase, case_dir: Path) -> bytes | None: return rendered if recorded_sha256 is not None else None +def _drop_verdict(case_dir: Path, verdict: bytes) -> None: + """Re-open a case: remove ``verdict``, the verdict prepare_audit read, + and its sidecar. + + The sidecar goes only when it describes ``verdict`` (its + ``verdict_sha256``) or records no verdict_sha256, as sidecars from before + runners recorded one do; a re-judge by the other runner must not inherit + it. Any other sidecar is the record of a verdict a runner is publishing, + which runners do sidecar first: removing it would leave that verdict with + no record of the bytes its judge read, and runners remove a sidecar left + without a verdict themselves. Likewise verdict.json goes only while it + still holds ``verdict``: one a runner has published since is left for the + collect gates and the next preparation to judge. + """ + verdict_path = case_dir / "verdict.json" + if _read_or_none(verdict_path) == verdict: + verdict_path.unlink(missing_ok=True) + meta = _sidecar(case_dir) + if meta is None: + return + recorded = meta.get("verdict_sha256") + if recorded is None or recorded == hashlib.sha256(verdict).hexdigest(): + (case_dir / "verdict.meta.json").unlink(missing_ok=True) + + def _sidecar(case_dir: Path) -> dict | None: """A case's ``verdict.meta.json``, or None when it has none. A sidecar that is not a JSON object records no template version. """ - path = case_dir / "verdict.meta.json" - if not path.is_file(): + text = _read_or_none(case_dir / "verdict.meta.json") + if text is None: return None try: - meta = json.loads(path.read_text()) + meta = json.loads(text) except ValueError: meta = None return meta if isinstance(meta, dict) else {TEMPLATE_VERSION_FIELD: None} diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 2f33388e..a07b550d 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -642,8 +642,11 @@ classify_one() { [ -f "$prompt" ] || return 0 case_ok "$out" "$name" 2>/dev/null && return 0 # No valid verdict: any sidecar left behind describes a verdict that no - # longer exists (re-prepared case) and must not outlive it. - rm -f "$tmp" "$meta_tmp" "$envelope" "$case_dir/verdict.meta.json" \ + # longer exists (re-prepared case) and must not outlive it. An invalid + # verdict.json goes too, so no verdict sits beside this run's sidecar until + # this run's verdict is published: audit-prepare removes a sidecar only + # together with the verdict it describes. + rm -f "$tmp" "$meta_tmp" "$envelope" "$out" "$case_dir/verdict.meta.json" \ "$case_dir/claude.transcript.jsonl" work=$(mktemp -d "${TMPDIR:-/tmp}/pb-judge.XXXXXX") || { echo "[FAIL] $(basename "$case_dir") (no scratch directory)" diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index 276eaa51..b2a60dfa 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -76,8 +76,11 @@ classify_one() { # temp file and publish atomically only once it validates, so an interrupted # run never leaves a half-written verdict that looks done. Default reasoning # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar - # left from a previous verdict describes a verdict that no longer exists. - rm -f "$tmp" "$case_dir/verdict.meta.json" + # left from a previous verdict describes a verdict that no longer exists, + # and an invalid verdict.json goes too, so no verdict sits beside this + # run's sidecar until this run's verdict is published: audit-prepare + # removes a sidecar only together with the verdict it describes. + rm -f "$tmp" "$out" "$case_dir/verdict.meta.json" # Codex reads a private copy, so the sidecar describes the bytes it judged # even if audit-prepare rewrites prompt.md meanwhile. judged=$(mktemp "${TMPDIR:-/tmp}/pb-codex-prompt.XXXXXX") || { diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 8a40f19e..8e891054 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -588,18 +588,23 @@ def test_codex_publishes_no_verdict_for_a_prompt_rewritten_while_it_judged( @pytest.mark.skipif(sys.platform == "win32", reason="bash runners") @pytest.mark.parametrize("runner", ["run_audit_claude.sh", "run_audit_codex.sh"]) +@pytest.mark.parametrize("earlier", [None, "{"]) def test_an_interrupted_publish_never_leaves_a_verdict_without_its_sidecar( - tmp_path: Path, runner + tmp_path: Path, runner, earlier ): """A worker killed between publishing its two files leaves at most a sidecar without a verdict, never a verdict that audit-prepare would read as a legacy v1 verdict with no record of the bytes its judge read. Here - the first move succeeds and the worker is then terminated. The next run - judges the case again and publishes both.""" + the first move succeeds and the worker is then terminated. A case that + held an invalid verdict (``earlier``) loses it when the run starts, so + no verdict sits beside the new sidecar either. The next run judges the + case again and publishes both.""" audit_dir, case_dir, env = _claude_audit(tmp_path) (case_dir / "prompt.md").write_text( template_header(1) + "\nCOUNTRY: US\nClassify this miss.\n" ) + if earlier is not None: + (case_dir / "verdict.json").write_text(earlier) shim = tmp_path / "shim" shim.mkdir() _fake_cli(shim / "mv", '/bin/mv "$@" || exit $?\nkill -TERM "$PPID"\n') @@ -654,3 +659,74 @@ def test_a_terminated_codex_worker_removes_its_prompt_copy(tmp_path: Path): assert list(copies.iterdir()) == [] assert not (case_dir / "verdict.json").exists() assert not (case_dir / "verdict.meta.json").exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_an_audit_prepare_between_a_publishs_two_moves_strands_nothing( + tmp_path: Path, +): + """End to end, the round-3 review's schedule on the real Codex runner: a + $250 case holding a truncated verdict is judged, and right after the + runner moves its sidecar into place an audit-prepare re-runs m1 at $999 + on v1; then the runner moves its verdict. The verdict keeps its sidecar, + whose prompt hash is the $250 prompt's: audit-collect refuses it and the + next audit-prepare re-opens it.""" + from policybench.audit import prepare_audit + + board = tmp_path / "us" + board.mkdir() + + def write_board(answer: float) -> None: + (board / "reference_outputs.csv").write_text( + "scenario_id,variable,value\ns0,snap,0.0\n" + ) + (board / "predictions.csv").write_text( + "model,scenario_id,variable,prediction,explanation,error\n" + f"m1,s0,snap,{answer},Estimated {answer}.,\n" + ) + + write_board(250.0) + audit_dir = tmp_path / "audit" + (case,) = prepare_audit(board, audit_dir, template_version=1) + case_dir = audit_dir / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + (case_dir / "verdict.json").write_text("{") + _, _, env = _claude_audit(tmp_path / "fixture") + hook = tmp_path / "prepare_999.py" + hook.write_text( + "from pathlib import Path\n" + "from policybench.audit import prepare_audit\n" + f"board = Path({str(board)!r})\n" + "(board / 'predictions.csv').write_text(\n" + " 'model,scenario_id,variable,prediction,explanation,error\\n'\n" + " 'm1,s0,snap,999.0,Estimated 999.0.,\\n'\n" + ")\n" + f"prepare_audit(board, Path({str(audit_dir)!r}), template_version=1)\n" + ) + shim = tmp_path / "shim" + shim.mkdir() + _fake_cli( + shim / "mv", + '/bin/mv "$@" || exit $?\n' + 'case "$3" in *verdict.meta.json)\n' + f' PYTHONPATH="{ROOT}" "{sys.executable}" "{hook}" || exit 1 ;;\n' + "esac\n", + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "PATH": f"{shim}:{env['PATH']}"}, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + assert f"[ok] {case_dir.name}" in result.stdout, result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + verdict = (case_dir / "verdict.json").read_bytes() + assert meta["verdict_sha256"] == hashlib.sha256(verdict).hexdigest() + assert meta["prompt_sha256"] == hashlib.sha256(judged).hexdigest() + assert b"999" in (case_dir / "prompt.md").read_bytes() + assert [c for c, _ in prompt_hash_problems(audit_dir)] == [case_dir.name] + prepare_audit(board, audit_dir, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py index 811630f8..0c1d9f33 100644 --- a/tests/test_judge_template.py +++ b/tests/test_judge_template.py @@ -677,6 +677,75 @@ def test_collect_refuses_a_verdict_published_on_rewritten_bytes(tmp_path): assert prompt_hash_problems(audit) == [] +def test_prepare_racing_a_publish_never_strands_the_verdict(tmp_path): + """The round-3 review's schedule, step by step. A $250 case holds a + truncated verdict.json. A runner's judge reads the $250 v1 prompt and the + runner publishes its sidecar. Before its verdict arrives, an + audit-prepare re-runs m1 at $999 on v1: it drops the truncated verdict + but leaves the sidecar, which describes another verdict. Then the + runner's verdict arrives. It still has its sidecar, whose prompt hash is + the $250 prompt's, so audit-collect refuses it and the next preparation + re-opens it.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + (case_dir / "verdict.json").write_text("{") + verdict = json.dumps(VERDICT, indent=2, sort_keys=True).encode() + sidecar = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": _sha256(verdict), + "prompt_sha256": _sha256(judged), + TEMPLATE_VERSION_FIELD: 1, + } + (case_dir / "verdict.meta.json").write_text(json.dumps(sidecar)) + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert json.loads((case_dir / "verdict.meta.json").read_text()) == sidecar + (case_dir / "verdict.json").write_bytes(verdict) + assert [case_id for case_id, _ in prompt_hash_problems(audit)] == [case.case_id] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + + +def test_a_dropped_verdict_takes_only_its_own_sidecar(tmp_path): + """Re-opening a case removes its sidecar when the sidecar describes the + verdict (verdict_sha256) or records no verdict_sha256, as legacy sidecars + do; a sidecar describing another verdict stays for its runner.""" + for name, verdict_sha256, stays in ( + ("own", "verdict", False), + ("legacy", None, False), + ("another verdict's", "0" * 64, True), + ): + root = tmp_path / name.replace(" ", "-").replace("'", "") + board, audit, case, case_dir = _judged_case(root) + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + if verdict_sha256 is None: + del meta["verdict_sha256"] + elif verdict_sha256 != "verdict": + meta["verdict_sha256"] = verdict_sha256 + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists(), name + assert (case_dir / "verdict.meta.json").exists() == stays, name + + +def test_a_verdict_its_sidecar_does_not_describe_is_reopened(tmp_path): + """An unchanged case whose verdict.json is not the verdict its sidecar + describes is re-opened; the sidecar, another verdict's, stays.""" + board, audit, case, case_dir = _judged_case(tmp_path) + (case_dir / "verdict.json").write_text(json.dumps({**VERDICT, "rationale": "x"})) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "verdict.meta.json").exists() + + def test_prompts_compare_and_keep_exact_bytes(tmp_path): """Byte equality, with no newline translation either way: a case whose text holds a carriage return keeps its verdict and its prompt's bytes on @@ -917,11 +986,12 @@ def test_each_past_release_driver_renders_on_v1_by_name(): st.sampled_from(JUDGED_STATES), st.sampled_from(PROMPT_KINDS), st.sampled_from(HASHES), + st.booleans(), st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), TEXT, ) def test_a_verdict_is_kept_on_its_recorded_version_alone( - state, kind, hashed, version, text + state, kind, hashed, described, version, text ): """Never inferred, and byte for byte: prepare_audit keeps an unchanged judged case's verdict exactly when the case rendered on the version its @@ -929,9 +999,11 @@ def test_a_verdict_is_kept_on_its_recorded_version_alone( which must also hash to the sidecar's prompt_sha256 when it records one; without prompt.md, that hash alone. A prompt.md on another version is re-opened, though template_version_of reads a version off it and though - the caller may name that very version, and so is a CRLF copy. A kept - case's files are untouched; a re-opened case renders on the caller's - version.""" + the caller may name that very version, and so is a CRLF copy. A sidecar + must also describe the verdict (its verdict_sha256); when ``described`` + is false, verdict.json is another verdict. A kept case's files are + untouched. A re-opened case loses its verdict, and its sidecar unless the + sidecar describes another verdict; it renders on the caller's version.""" with tempfile.TemporaryDirectory() as scratch: root = Path(scratch) board = _board(root / "us", {"s0": 250.0}) @@ -964,22 +1036,29 @@ def test_a_verdict_is_kept_on_its_recorded_version_alone( ) sidecar = {**sidecar, "prompt_sha256": recorded_sha256} _judge(case_dir, sidecar) + if not described: + (case_dir / "verdict.json").write_text( + json.dumps({**VERDICT, "rationale": "Another verdict."}) + ) + other_verdict = sidecar is not None and not described before = _tree(audit) recorded = recorded_template_version(sidecar) judged = renders.get(recorded) kept = ( - judged is not None + not other_verdict + and judged is not None and recorded_sha256 in (None, _sha256(judged)) and (prompt == judged if prompt is not None else bool(recorded_sha256)) ) prepare_audit(board, audit, template_version=version) after = _tree(audit) + where = (state, kind, hashed, described) if kept: restored = {f"cases/{case.case_id}/prompt.md": judged} - assert after == {**before, **restored}, (state, kind, hashed) + assert after == {**before, **restored}, where else: - assert not (case_dir / "verdict.json").exists(), (state, kind, hashed) - assert not (case_dir / "verdict.meta.json").exists() + assert not (case_dir / "verdict.json").exists(), where + assert (case_dir / "verdict.meta.json").exists() == other_verdict, where assert (case_dir / "prompt.md").read_bytes() == renders[version] assert template_version_problems(audit) == [] assert prompt_hash_problems(audit) == [] From 80bad56e7941d9c61995b734ee589f67d29965af Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:45:08 -0400 Subject: [PATCH 7/8] Refuse and re-open a verdict that has no sidecar Answers review round 4 of #210: _drop_verdict can still remove a sidecar a runner has just published (it reads, then unlinks, the path; and a new verdict whose bytes repeat the old one's matches the old digest), and the stranded verdict passed as legacy v1. Every such schedule ends with a runner's verdict without its sidecar, so that state is no longer accepted: recorded_template_version(None) is None, prepare_audit re-opens the verdict, and template_version_problems (so audit-collect) refuses it. A sidecar without the field still means v1; every verdict in releases 20260929 and 20260930 has a sidecar. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 17 +++++++++++----- policybench/audit.py | 18 ++++++++++------- policybench/judge_template.py | 16 +++++++++------ tests/test_judge_template.py | 37 +++++++++++++++++++++++++++++++---- 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index 504e3c6f..8be3c77f 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -80,9 +80,14 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` publishes the sidecar before the verdict. An interrupted publish can leave a sidecar without a verdict, which the next run removes, but never a new verdict without its sidecar. -- A sidecar without the field, or a verdict without a sidecar, was judged on - v1 (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars - written before versions existed, not a guess from the prompt. +- A sidecar without the field was judged on v1 + (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars written + before versions existed, not a guess from the prompt. +- A verdict without a sidecar has no record of what its judge read. + `audit-prepare` re-opens it and `audit-collect` refuses it. A runner's + verdict can lose its sidecar to an `audit-prepare` racing it, so this is + what keeps such a verdict out. Every verdict in releases 20260929 and + 20260930 has a sidecar. - `audit-prepare` renders a case that already has a verdict with the version its sidecar records. The verdict stands only if that reproduces `prompt.md` byte for byte and, when the sidecar records `prompt_sha256` @@ -102,8 +107,10 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` bytes its judge did not read. That holds because a runner never leaves a new verdict without its sidecar, `audit-prepare` never removes another verdict's sidecar, and `audit-collect` checks each recorded hash. A - verdict judged before a racing `audit-prepare` rewrote its case is - refused, not kept. Run `audit-collect` after both finish. + verdict that ends up without its sidecar all the same, in whatever + interleaving, is refused and re-opened. A verdict judged before a racing + `audit-prepare` rewrote its case is refused, not kept. Run `audit-collect` + after both finish. - Each release driver names the version its new and re-opened cases render on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, diff --git a/policybench/audit.py b/policybench/audit.py index 0e60044f..4ef68e98 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -421,8 +421,8 @@ def _judged_prompt(case: AuditCase, case_dir: Path, verdict: bytes) -> bytes | N A sidecar that records ``verdict_sha256`` must describe ``verdict``: otherwise it is another verdict's record, such as one a runner has published and whose verdict has not arrived yet. The case is rendered on - the version the sidecar records (absent: v1; an unknown version gives - None). Those bytes must be prompt.md's exactly, and, when the sidecar + the version the sidecar records (absent: v1; an unknown version, or no + sidecar, gives None). Those bytes must be prompt.md's exactly, and, when the sidecar records ``prompt_sha256`` (the hash of the bytes its judge read), hash to it. Without prompt.md, that hash is the only evidence of what the judge read, so a sidecar without one gives None. @@ -493,10 +493,12 @@ def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: A tree may mix versions: a release's carried seeds keep the version they were judged on, while its new and re-opened cases use the one its driver - names. Every verdict must have its prompt.md, its sidecar must name a - known version (absent: v1), and prompt.md must begin with that version's - header. Returns ``(case_id, problem)`` for each case that fails, in case - order. + names. Every verdict must have its sidecar and its prompt.md, its + sidecar must name a known version (absent: v1), and prompt.md must begin + with that version's header. A verdict without a sidecar has no record of + the bytes its judge read: one a runner published can lose its sidecar to + an audit-prepare racing it. Returns ``(case_id, problem)`` for each case + that fails, in case order. """ problems: list[tuple[str, str]] = [] cases_root = audit_dir / "cases" @@ -508,7 +510,9 @@ def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: meta = _sidecar(case_dir) recorded = recorded_template_version(meta) prompt_path = case_dir / "prompt.md" - if recorded is None: + if meta is None: + problem = "a verdict without its sidecar (verdict.meta.json)" + elif recorded is None: problem = ( f"its sidecar's {TEMPLATE_VERSION_FIELD} " f"{meta.get(TEMPLATE_VERSION_FIELD)!r} names no template " diff --git a/policybench/judge_template.py b/policybench/judge_template.py index 2b55c773..04d5d7f5 100644 --- a/policybench/judge_template.py +++ b/policybench/judge_template.py @@ -19,8 +19,9 @@ Each verdict's provenance sidecar (``verdict.meta.json``) records the version of the prompt it was judged on as ``judge_template_version``. A sidecar -without the field, or a verdict without a sidecar, predates the versions and -was judged on v1. +without the field predates the versions and was judged on v1. A verdict +without a sidecar has no record of what its judge read, so it has no +recorded version: audit-prepare re-opens it and audit-collect refuses it. Version selection is explicit: every renderer (:func:`policybench.audit.render_case_prompt`, @@ -143,11 +144,14 @@ def template_version_of(prompt: str | bytes) -> int | None: def recorded_template_version(meta: Mapping | None) -> int | None: """The version a verdict's sidecar records it was judged on. - No sidecar (None), or one without the field, means v1. A value that names - no version (null, a string, a boolean, an unknown number) gives None: the - verdict's template is unknown. + A sidecar without the field means v1: it predates the versions. No + sidecar (None) records nothing, so it gives None, and so does a value + that names no version (null, a string, a boolean, an unknown number): + the verdict's template is unknown. """ - if meta is None or TEMPLATE_VERSION_FIELD not in meta: + if meta is None: + return None + if TEMPLATE_VERSION_FIELD not in meta: return UNRECORDED_TEMPLATE_VERSION value = meta[TEMPLATE_VERSION_FIELD] if type(value) is int and value in JUDGE_TEMPLATE_HEADERS: diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py index 0c1d9f33..8b86f41e 100644 --- a/tests/test_judge_template.py +++ b/tests/test_judge_template.py @@ -176,7 +176,8 @@ def test_recorded_version_is_the_field_when_it_names_one(value): expected = value if type(value) is int and value in JUDGE_TEMPLATE_HEADERS else None assert recorded_template_version({TEMPLATE_VERSION_FIELD: value}) == expected assert recorded_template_version({"judge_runner": "x"}) == 1 - assert recorded_template_version(None) == 1 + # No sidecar records nothing. + assert recorded_template_version(None) is None # --- Rendering ------------------------------------------------------------------- @@ -484,7 +485,7 @@ def _mixed_tree(tmp_path: Path) -> tuple[Path, Path, dict[str, AuditCase]]: audit, board, { - "s0": "no sidecar", + "s0": "v1, field absent", "s1": "v1, field absent", "s2": "v1", "s3": "v2", @@ -536,7 +537,7 @@ def meta(scenario: str, **fields) -> None: sidecar.write_text(json.dumps(stripped)) # v2 prompt, field absent (v1) meta("s2", judge_template_version="2") # names no version (case_dir["s0"] / "prompt.md").unlink() # a verdict without its prompt - (case_dir["s4"] / "verdict.json").write_text(json.dumps(VERDICT)) + _judge(case_dir["s4"], {}) # judged, sidecar without the field (v1) (case_dir["s4"] / "prompt.md").write_text("Classify this miss.\n") problems = dict(template_version_problems(audit)) assert problems == { @@ -556,7 +557,8 @@ def meta(scenario: str, **fields) -> None: _collect_cli(board, audit, tmp_path / "out") assert not (tmp_path / "out").exists() # Preparing again re-opens every flagged case. The verdict that lost its - # prompt has no sidecar, so nothing records the bytes its judge read. + # prompt has no prompt_sha256, so nothing records the bytes its judge + # read. prepare_audit(board, audit, template_version=2) assert template_version_problems(audit) == [] for scenario in ("s0", "s1", "s2", "s3", "s4"): @@ -713,6 +715,33 @@ def test_prepare_racing_a_publish_never_strands_the_verdict(tmp_path): assert not (case_dir / "verdict.meta.json").exists() +def test_a_verdict_without_its_sidecar_is_refused_and_reopened(tmp_path): + """The round-4 review's schedule ends with a runner's verdict and no + sidecar. A $250 case's verdict is dropped by an audit-prepare re-running + m1 at $999, which pauses and then removes the sidecar the runner has just + published (the old sidecar matched the runner's verdict, whose bytes + repeat the old verdict's); then the runner's verdict arrives. Nothing + records what its judge read, so audit-collect refuses it and the next + audit-prepare re-opens it.""" + board, audit, case, case_dir = _judged_case(tmp_path) + verdict = (case_dir / "verdict.json").read_bytes() + _board(board, {"s0": 999.0}) + (case_dir / "verdict.json").unlink() + (case_dir / "verdict.meta.json").unlink() + (case_dir / "prompt.md").write_bytes( + render_case_prompt(build_audit_cases(board)[0], template_version=1).encode() + ) + (case_dir / "verdict.json").write_bytes(verdict) + problem = "a verdict without its sidecar (verdict.meta.json)" + assert template_version_problems(audit) == [(case.case_id, problem)] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert template_version_problems(audit) == [] + + def test_a_dropped_verdict_takes_only_its_own_sidecar(tmp_path): """Re-opening a case removes its sidecar when the sidecar describes the verdict (verdict_sha256) or records no verdict_sha256, as legacy sidecars From e23e33cc7747cf0b8977521be685272e0057ad7d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:04:21 -0400 Subject: [PATCH 8/8] State the sidecar race as narrowed, not closed Answers review round 5 of #210 (APPROVE, one non-blocking note): docs/audit.md said audit-prepare never removes another verdict's sidecar, which round 4's schedule disproves. The guarantee rests on refusing and re-opening a verdict without its sidecar and on the prompt-hash check; the docs and _drop_verdict's docstring now say so. No code change. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 25 +++++++++++++++---------- policybench/audit.py | 5 +++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index 420ab4cb..8f8998c5 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -98,19 +98,24 @@ adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/` was judged, or its version or judged bytes are unknown, and the case is re-opened: the verdict is dropped, and so is its sidecar when the sidecar describes that verdict or records no `verdict_sha256`. A sidecar that - describes another verdict stays, such as the record of a verdict a runner - is still publishing. A `prompt.md` on another version is re-opened, not - adopted. New and re-opened cases use the version the caller names. + describes another verdict when `audit-prepare` reads it, such as the + record of a verdict a runner is still publishing, is left in place. A + `prompt.md` on another version is re-opened, not adopted. New and + re-opened cases use the version the caller names. - `audit-prepare` may run while a runner judges the same tree, with one runner per case and one `audit-prepare` at a time. Once both finish, no verdict passes `audit-collect`, or survives the next `audit-prepare`, on - bytes its judge did not read. That holds because a runner never leaves a - new verdict without its sidecar, `audit-prepare` never removes another - verdict's sidecar, and `audit-collect` checks each recorded hash. A - verdict that ends up without its sidecar all the same, in whatever - interleaving, is refused and re-opened. A verdict judged before a racing - `audit-prepare` rewrote its case is refused, not kept. Run `audit-collect` - after both finish. + bytes its judge did not read. Two rules carry that guarantee. First, a + verdict without its sidecar is refused and re-opened. Second, a sidecar + that survives is the runner's own record of the bytes its judge read, so + a verdict whose case `audit-prepare` rewrote meanwhile fails the hash + check: `audit-collect` refuses it and `audit-prepare` re-opens it. The + other measures make a stranded verdict rare without ruling it out. + Runners start each case clean and publish the sidecar first, and + `audit-prepare` removes a sidecar only when it describes the verdict being + dropped. But `audit-prepare` reads a sidecar before it removes it, so a + runner publishing in between can still lose its sidecar; its verdict is + then refused. Run `audit-collect` after both finish. - Each release driver names the version its new and re-opened cases render on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, diff --git a/policybench/audit.py b/policybench/audit.py index 4ef68e98..a2699761 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -461,6 +461,11 @@ def _drop_verdict(case_dir: Path, verdict: bytes) -> None: without a verdict themselves. Likewise verdict.json goes only while it still holds ``verdict``: one a runner has published since is left for the collect gates and the next preparation to judge. + + This narrows the race without closing it: the sidecar is read before it + is removed, so a runner publishing in between can still lose its own. + Such a verdict has no sidecar, so _judged_prompt re-opens it and + template_version_problems refuses it. """ verdict_path = case_dir / "verdict.json" if _read_or_none(verdict_path) == verdict: