diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index 26e0b045..c4a0707d 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -46,7 +46,25 @@ PolicyBench's diagnosis judge (`docs/audit.md`) explains why models miss. Its pr Inputs: -- Payload: `data.json.gz` of release dashboard-data-20260930, as committed at `8b4c0ca1` (#187), sha256 `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18`. Release dashboard-data-20261006 (#202) later rewrote the working tree's payload and annotations; the tests and the Reproduce steps read the pass's inputs from `8b4c0ca1`. +- The frozen run and its annotations, as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Release dashboard-data-20261006 (#202) later rewrote the working tree's payload, exclusion record and annotations, so reproducing the pass needs each file below as `8b4c0ca1` holds it, with that sha256. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + + | Input at `8b4c0ca1` | sha256 | Read by | + |---|---|---| + | `/data.json.gz`, the payload | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | `consensus-flags`, `adversary-prepare`, `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/predictions.csv.gz` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | `leaderboard_impact.py` | + | `/reference_outputs.csv` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/reference_outputs.csv.meta.json` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/reference_exclusions.json` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/scenarios.csv` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | + | `/scenarios.csv.meta.json` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | + | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | + +- `proposed_changes.json` as #200 merged it at `4db91b5f`, sha256 `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf`, read by `leaderboard_impact.py`. +- Who checks the pins: + - `leaderboard_impact.py` stages every input it reads from git and stops on any sha256 mismatch before scoring. + - `definition_conformance.py` and `publication_sources.py` stop when the working tree's payload is not `1e029aaa…`, but read their other inputs from the working tree unchecked. + - `engine_probe.py` and `build_proposals.py` check no input's sha256. + - `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. `tests/test_reference_adversary_impact.py` checks the impact script's pins and that it ignores the working tree's run and proposals. - Top 5 by the payload's `modelStats` order: gpt-6-sol, claude-opus-5.5, gpt-5.6-sol, gpt-6-luna and claude-sonnet-5.5. - `consensus_flags.json` holds the default run, and `consensus_flags_prototype.json` reproduces the 2026-10-05 prototype. - `verification/flags_table.md` lists all 61 cells, each with its cluster, its exact-model count and whether the prototype flagged it. @@ -286,7 +304,7 @@ The root cause is a gap in scenario generation and definition scope, not an engi ## Leaderboard impact -`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. +`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. It takes the run and the proposals from git at their pinned commits (Inputs above), not from the working tree. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. | Variant | Scored cells | Exact-rate change across models | Leader (gpt-6-sol) exact | Exact-rank changes | |---|---:|---|---|---| @@ -443,11 +461,18 @@ The pass's own expectations are not part of either ruling (`status.not_ruled`). | `proposed_changes.json` | Exclusion and regeneration records per root cause, and the rulings | | `scripts/` | The scripts that wrote the files above | -The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. +The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_impact.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. ## Reproduce -`/data.json.gz` and `annotations//` must hold the pass's inputs as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. A worktree at `8b4c0ca1` has them, or `git show 8b4c0ca1:` each file. +Every step reads the pass's inputs (the table under [Flag parameters](#flag-parameters)) as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. `scripts/leaderboard_impact.py` stages its own inputs from git, so it runs from any checkout that has `8b4c0ca1` and `4db91b5f` in its history. The commands below read the payload and annotations at the paths they are given. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `build_proposals.py` read the working tree's `/`. So first restore every file in the table (the run directory and the run's annotations directory) from `8b4c0ca1`, and put today's back when done: + +```bash +run=paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace +git restore --source 8b4c0ca1 -- "$run" "annotations/${run##*/}" +# ... the steps below ... +git restore -- "$run" "annotations/${run##*/}" +``` ```bash uv run policybench consensus-flags --payload /data.json.gz \ @@ -481,3 +506,9 @@ uv run python reference_audit/2026-10-05-reference-adversary/scripts/search_expo Preparing `runs/claude` today reproduces its `cases.jsonl` and derivations byte for byte. Each stage-1 prompt then differs from the committed one in one line, because the source rule now asks for `blocked_domains`. The re-judge's preparation reproduces `runs/claude-rejudge`'s inputs byte for byte. The scripts in `scripts/` regenerate the verification files; each script's docstring gives its command. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment. + +`tests/test_reference_adversary_impact.py` regenerates the `verification/leaderboard_impact*` files into a scratch directory and requires the committed ones byte for byte. The test takes about five minutes, so it is marked slow and CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_impact.py +``` diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py index d728c5ee..23439c69 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py @@ -1,9 +1,19 @@ """Score the frozen run as published and under each alternative in proposed_changes.json. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch -directories and never writes the snapshot. Each copy is scored with -``python -m policybench.cli analyze``, the command the freeze runs (the method of -PolicyEngine/policybench#191 and #194): +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as commit PASS_COMMIT +(release dashboard-data-20260930, #187) holds it, and proposed_changes.json as +PROPOSALS_COMMIT (#200) holds it. Later releases rewrite the working tree's run +(dashboard-data-20261006, #202, rewrote its payload and exclusions), and scoring +those would regenerate different evidence into this pass's verification files. +Each file is staged under ``/pass_inputs`` and must match its pinned +sha256, or the script stops before scoring anything. The scoring code is the +checkout's; ``published`` below stops the script if that code no longer scores +the pinned run as the pinned payload records. + +Copies the staged run to scratch directories and never writes the snapshot. Each +copy is scored with ``python -m policybench.cli analyze``, the command the freeze +runs (the method of PolicyEngine/policybench#191 and #194): ``published`` Unchanged. It must reproduce the published payload's scoring: every modelStats field @@ -24,6 +34,11 @@ PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py \\ --scratch + +The checkout needs PASS_COMMIT and PROPOSALS_COMMIT in its history (a shallow +clone needs ``git fetch --unshallow``). ``--out-dir`` writes the verification +files elsewhere; tests/test_reference_adversary_impact.py regenerates them that +way and requires the committed ones byte for byte. """ from __future__ import annotations @@ -42,12 +57,36 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -PROPOSALS = HERE / "proposed_changes.json" OUT_DIR = HERE / "verification" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +RUN_SHA256 = { + "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "predictions.csv.gz": ( + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12" + ), + "reference_outputs.csv": ( + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" + ), + "reference_outputs.csv.meta.json": ( + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" + ), + "reference_exclusions.json": ( + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" + ), + "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +PROPOSALS_COMMIT = "4db91b5f10581947f60a07f899e0ec2867b0405e" +PROPOSALS_PATH = "reference_audit/2026-10-05-reference-adversary/proposed_changes.json" +PROPOSALS_SHA256 = "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf" +PROPOSALS_NAME = Path(PROPOSALS_PATH).name +# Copied into each variant and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -74,13 +113,46 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def proposals() -> dict: - return json.loads(PROPOSALS.read_text())["root_causes"] +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target -def variants() -> dict[str, dict[str, list[dict]]]: - """Each variant's exclusion and regeneration records.""" - causes = proposals() +def pass_inputs(target: Path) -> Path: + """Stage the pass's run bundle under ``target/run`` and its proposals beside it.""" + if target.exists(): + shutil.rmtree(target) + for name, pinned in RUN_SHA256.items(): + git_input(PASS_COMMIT, f"{RUN_PATH}/{name}", pinned, target / "run" / name) + git_input( + PROPOSALS_COMMIT, PROPOSALS_PATH, PROPOSALS_SHA256, target / PROPOSALS_NAME + ) + return target + + +def proposals(path: Path) -> dict: + return json.loads(path.read_text())["root_causes"] + + +def variants(path: Path) -> dict[str, dict[str, list[dict]]]: + """Each variant's exclusion and regeneration records, from ``path``'s proposals.""" + causes = proposals(path) plan: dict[str, dict[str, list[dict]]] = {} for cause, spec in causes.items(): plan[f"exclude:{cause}"] = {"exclude": spec["exclusions"], "regenerate": []} @@ -140,12 +212,13 @@ def always_zero(run_dir: Path) -> dict[str, float]: } -def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: +def stage(source: Path, target: Path, records: dict[str, list[dict]] | None) -> Path: + """Copy the staged run at ``source`` to ``target`` and apply ``records``.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if not records: return target if records["exclude"]: @@ -165,7 +238,7 @@ def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: raise SystemExit(f"{key}: frozen value differs from the reference CSV") reference.at[row, "value"] = repr(float(change["regenerated_value"])) reference.to_csv(target / "reference_outputs.csv", index=False) - before = (RUN / "reference_outputs.csv").read_text().splitlines() + before = (source / "reference_outputs.csv").read_text().splitlines() after = (target / "reference_outputs.csv").read_text().splitlines() changed = [i for i, (a, b) in enumerate(zip(before, after)) if a != b] if len(before) != len(after) or len(changed) != len(changes): @@ -211,8 +284,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -331,27 +405,36 @@ def main() -> None: parser.add_argument( "--only", nargs="*", help="score only these variants (default: all)" ) + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() scratch = Path(args.scratch).resolve() - # stage() deletes and rewrites scratch/; never inside the repository. + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/; never inside + # the repository. if scratch == ROOT.resolve() or scratch.is_relative_to(ROOT.resolve()): parser.error(f"--scratch must be outside the repository ({ROOT})") - plan = variants() + inputs = pass_inputs(scratch / "pass_inputs") + source = inputs / "run" + plan = variants(inputs / PROPOSALS_NAME) unknown = sorted(set(args.only or ()) - set(plan)) if unknown: parser.error(f"unknown variant(s) {unknown}; choose from {sorted(plan)}") - base_dir = stage(scratch / "published", None) + base_dir = stage(source, scratch / "published", None) base = analyze(base_dir) - check_reproduces_published(base) + check_reproduces_published(base, source) zero_published = always_zero(base_dir) - OUT_DIR.mkdir(parents=True, exist_ok=True) - summary: dict = {"published_reproduced": True, "proposals": PROPOSALS.name} + out_dir.mkdir(parents=True, exist_ok=True) + summary: dict = {"published_reproduced": True, "proposals": PROPOSALS_NAME} if args.only: plan = {name: plan[name] for name in args.only} for name, records in plan.items(): - run_dir = stage(scratch / name.replace(":", "__"), records) + run_dir = stage(source, scratch / name.replace(":", "__"), records) other = analyze(run_dir) models = compare(base, other) programs = program_rows(base, other) @@ -363,8 +446,8 @@ def main() -> None: } ) slug = name.replace(":", "_") - models.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_models.csv") - programs.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_programs.csv") + models.to_csv(out_dir / f"leaderboard_impact_{slug}_models.csv") + programs.to_csv(out_dir / f"leaderboard_impact_{slug}_programs.csv") summary[name] = summarize(models, base, other, zero, cells) summary[name]["programs"] = programs.round(6).to_dict(orient="index") with pd.option_context("display.width", 250, "display.max_rows", 100): @@ -385,7 +468,7 @@ def main() -> None: ].round(4) ) print(programs.round(4)) - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(json.dumps({k: v for k, v in summary.items() if k != "proposals"}, indent=2)[:4000]) diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py new file mode 100644 index 00000000..612d9f42 --- /dev/null +++ b/tests/test_reference_adversary_impact.py @@ -0,0 +1,318 @@ +"""The reference adversary's leaderboard impact comes from the pass's pinned inputs. + +``reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py`` +reads the frozen run as PASS_COMMIT (release dashboard-data-20260930, #187) +committed it and the proposals as #200 merged them, each checked against its +sha256. It never reads the working tree's run, which later releases rewrite +(#202 did). CI checks out full history, so git holds every pinned input. +""" + +from __future__ import annotations + +import ast +import difflib +import gzip +import importlib.util +import json +import os +import re +import subprocess +import sys +import tempfile +from functools import cache +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" +SCRIPT = AUDIT / "scripts/leaderboard_impact.py" +VERIFICATION = AUDIT / "verification" +# The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" +# Release dashboard-data-20261006 (#202) rewrote the run's payload and exclusions. +RELEASE_20261006_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = ( + "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d" +) +COMMITTED = sorted(path.name for path in VERIFICATION.glob("leaderboard_impact*")) + + +def _load_script(): + spec = importlib.util.spec_from_file_location("adversary_impact", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +impact = _load_script() + + +def _tampered_checkout(path: Path) -> Path: + """A checkout sharing this repository's objects whose working tree holds junk. + + Its run directory and proposals are not the pass's inputs, so a script that + read them instead of git would fail or regenerate different evidence. + """ + subprocess.run( + ["git", "clone", "--quiet", "--shared", "--no-checkout", str(ROOT), str(path)], + check=True, + ) + for name in impact.RUN_SHA256: + junk = path / impact.RUN_PATH / name + junk.parent.mkdir(parents=True, exist_ok=True) + junk.write_bytes(b"not the pass's input\n") + proposals = path / impact.PROPOSALS_PATH + proposals.parent.mkdir(parents=True, exist_ok=True) + proposals.write_text('{"root_causes": {}}\n') + return path + + +def _argv(monkeypatch, tmp_path: Path) -> Path: + out = tmp_path / "out" + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + ) + return out + + +def test_the_pins_cover_every_file_the_script_reads(tmp_path): + assert impact.PASS_COMMIT == PASS_COMMIT + assert impact.RUN_SHA256["data.json.gz"] == FROZEN_SHA256 + assert set(impact.RUN_SHA256) == {*impact.RUN_FILES, "data.json.gz"} + + inputs = impact.pass_inputs(tmp_path / "pass_inputs") + staged = sorted( + path.relative_to(inputs).as_posix() + for path in inputs.rglob("*") + if path.is_file() + ) + assert staged == sorted( + [f"run/{name}" for name in impact.RUN_SHA256] + [impact.PROPOSALS_NAME] + ) + for name, pinned in impact.RUN_SHA256.items(): + assert impact.sha256(inputs / "run" / name) == pinned + assert impact.sha256(inputs / impact.PROPOSALS_NAME) == impact.PROPOSALS_SHA256 + # The README's frozen run: 46 models, 1,928 scored cells. + payload = json.loads(gzip.decompress((inputs / "run/data.json.gz").read_bytes())) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + (inputs / "stray").write_text("left over\n") + impact.pass_inputs(inputs) + assert not (inputs / "stray").exists() + + +def test_the_payload_pin_matches_the_engine_side_scripts_pins(): + """The engine-side scripts pin the same payload (read without importing them).""" + for name in ("definition_conformance.py", "publication_sources.py"): + tree = ast.parse((AUDIT / "scripts" / name).read_text()) + pins = [ + node.value.value + for node in tree.body + if isinstance(node, ast.Assign) + and [target.id for target in node.targets] == ["PAYLOAD_SHA256"] + ] + assert pins == [impact.RUN_SHA256["data.json.gz"]], name + + +@cache +def _pinned_bytes(name: str) -> bytes: + return subprocess.run( + ["git", "-C", str(ROOT), "show", f"{PASS_COMMIT}:{impact.RUN_PATH}/{name}"], + capture_output=True, + check=True, + ).stdout + + +@st.composite +def _edits(draw) -> tuple[str, str, int, int]: + """One pinned run file and one change to its bytes: flip, cut or extend.""" + name = draw(st.sampled_from(sorted(impact.RUN_SHA256))) + kind = draw(st.sampled_from(["flip", "truncate", "append"])) + position = draw(st.integers(0, len(_pinned_bytes(name)) - 1)) + return name, kind, position, draw(st.integers(1, 255)) + + +@settings(max_examples=40, deadline=None) +@given(_edits()) +def test_any_changed_byte_in_a_pinned_input_is_refused_unwritten(edit): + """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" + name, kind, position, byte = edit + original = _pinned_bytes(name) + if kind == "flip": + changed = bytearray(original) + changed[position] ^= byte + tampered = bytes(changed) + elif kind == "truncate": + tampered = original[:position] + else: + tampered = original + bytes([byte]) + shown = SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=tampered, stderr=b"" + ) + ) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / name + with mock.patch.object(impact, "subprocess", shown): + with pytest.raises(SystemExit, match="not the pinned"): + impact.git_input( + PASS_COMMIT, + f"{impact.RUN_PATH}/{name}", + impact.RUN_SHA256[name], + target, + ) + assert not target.exists() + # The unchanged bytes pass the same check. + unchanged = SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=original, stderr=b"" + ) + ) + with mock.patch.object(impact, "subprocess", unchanged): + impact.git_input( + PASS_COMMIT, + f"{impact.RUN_PATH}/{name}", + impact.RUN_SHA256[name], + target, + ) + assert target.read_bytes() == original + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ( + "PASS_COMMIT", + RELEASE_20261006_COMMIT, + f"data.json.gz has sha256 {RELEASE_20261006_SHA256}, " + f"not the pinned {FROZEN_SHA256}", + ), + ("PROPOSALS_SHA256", "0" * 64, f"not the pinned {'0' * 64}"), + ("PASS_COMMIT", "0" * 40, "cannot read"), + ], +) +def test_pass_inputs_refuse_anything_but_the_pinned_bytes( + tmp_path, monkeypatch, attribute, value, message +): + monkeypatch.setattr(impact, attribute, value) + with pytest.raises(SystemExit, match=re.escape(message)): + impact.pass_inputs(tmp_path / "pass_inputs") + + +def test_a_refused_input_stops_the_script_before_it_scores(tmp_path, monkeypatch): + def analyze(run_dir): + raise AssertionError(f"scored {run_dir} after its inputs were refused") + + monkeypatch.setattr(impact, "PASS_COMMIT", RELEASE_20261006_COMMIT) + monkeypatch.setattr(impact, "analyze", analyze) + out = _argv(monkeypatch, tmp_path) + with pytest.raises(SystemExit, match=f"not the pinned {FROZEN_SHA256}"): + impact.main() + assert not out.exists() + + +def test_the_script_scores_only_the_staged_inputs(tmp_path, monkeypatch): + """Run from a checkout whose working-tree run is junk, every variant still + starts from the pinned bytes and the full set of evidence files is written.""" + monkeypatch.setattr(impact, "ROOT", _tampered_checkout(tmp_path / "checkout")) + scored: dict[str, dict[str, str]] = {} + published: dict = {} + + def analyze(run_dir: Path) -> dict: + scored[run_dir.name] = { + name: impact.sha256(run_dir / name) for name in impact.RUN_FILES + } + if not published: + payload = run_dir.parent / "pass_inputs/run/data.json.gz" + published.update(json.loads(gzip.decompress(payload.read_bytes()))) + return published + + monkeypatch.setattr(impact, "analyze", analyze) + monkeypatch.setattr( + impact, "always_zero", lambda run_dir: {"exact": 0.0, "within1pct": 0.0} + ) + out = _argv(monkeypatch, tmp_path) + impact.main() + + pinned = {name: impact.RUN_SHA256[name] for name in impact.RUN_FILES} + assert scored.pop("published") == pinned + exclusions = {"reference_exclusions.json"} + regenerations = {"reference_outputs.csv", "reference_outputs.csv.meta.json"} + expected = { + "recommended": exclusions | regenerations, + "exclude_all": exclusions, + } + assert set(scored) >= set(expected) + for variant, digests in scored.items(): + changed = {name for name in pinned if digests[name] != pinned[name]} + if variant.startswith("exclude__"): + assert changed == exclusions, variant + elif variant.startswith("regenerate__"): + assert changed == regenerations, variant + else: + assert changed == expected[variant], variant + assert sorted(path.name for path in out.iterdir()) == COMMITTED + summary = json.loads((out / "leaderboard_impact.json").read_text()) + assert summary["published_reproduced"] is True + assert summary["proposals"] == "proposed_changes.json" + + +@pytest.mark.slow +def test_regeneration_reproduces_the_committed_evidence(tmp_path): + """Score every variant for real and require the committed files byte for byte. + + About five minutes of ``policybench analyze`` runs, so CI deselects it. Run it + with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_adversary_impact.py + """ + assert len(COMMITTED) == 23 + out = tmp_path / "out" + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + cwd=ROOT, + env=dict(os.environ, PYTHONPATH=str(ROOT)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + assert sorted(path.name for path in out.iterdir()) == COMMITTED + differing = [ + name + for name in COMMITTED + if (out / name).read_bytes() != (VERIFICATION / name).read_bytes() + ] + if differing: + first = differing[0] + diff = difflib.unified_diff( + (VERIFICATION / first).read_text().splitlines(), + (out / first).read_text().splitlines(), + f"committed/{first}", + f"regenerated/{first}", + lineterm="", + ) + pytest.fail( + f"{len(differing)} files differ: {differing}\n" + "\n".join(list(diff)[:60]) + )