From bec3b244417e78f16b977a1bf54110099d5265cf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 21:34:50 -0400 Subject: [PATCH 01/18] Add the reference-adversary audit pass: consensus trigger, blind adversary prompt and runners, definition conformance, publication sources (WIP: adversary not yet run) Work from the reference-adversary session (task_8940406e), which stopped at the 2026-10-05 account cutoff before committing. 361 tests pass (test_consensus, test_definition_conformance, test_publication_sources, test_reference_adversary, test_reference_adversary_runner, test_audit). The consensus flags (61 of 1,928 cells), the definition-conformance report and the publication-source report are generated; the adversary judges have not run yet. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 89 + policybench/audit.py | 9 +- policybench/cli.py | 176 + policybench/consensus.py | 365 + policybench/definition_conformance.py | 1578 ++ policybench/publication_sources.py | 1906 ++ policybench/reference_adversary.py | 1962 ++ .../consensus_flags.json | 4716 ++++ .../consensus_flags_prototype.json | 3205 +++ .../covered_elsewhere.json | 56 + .../scripts/definition_conformance.py | 760 + .../scripts/flags_table.py | 68 + .../scripts/publication_sources.py | 1033 + .../verification/definition_conformance.json | 15164 +++++++++++++ .../verification/definition_conformance.md | 47 + .../verification/flags_table.md | 71 + .../verification/publication_sources.json | 18551 ++++++++++++++++ .../verification/publication_sources.md | 330 + scripts/run_reference_adversary_claude.sh | 392 + scripts/run_reference_adversary_codex.sh | 286 + tests/test_audit.py | 12 + tests/test_consensus.py | 838 + tests/test_definition_conformance.py | 1174 + tests/test_publication_sources.py | 1268 ++ tests/test_reference_adversary.py | 1476 ++ tests/test_reference_adversary_runner.py | 603 + 26 files changed, 56128 insertions(+), 7 deletions(-) create mode 100644 policybench/consensus.py create mode 100644 policybench/definition_conformance.py create mode 100644 policybench/publication_sources.py create mode 100644 policybench/reference_adversary.py create mode 100644 reference_audit/2026-10-05-reference-adversary/consensus_flags.json create mode 100644 reference_audit/2026-10-05-reference-adversary/consensus_flags_prototype.json create mode 100644 reference_audit/2026-10-05-reference-adversary/covered_elsewhere.json create mode 100644 reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py create mode 100644 reference_audit/2026-10-05-reference-adversary/scripts/flags_table.py create mode 100644 reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py create mode 100644 reference_audit/2026-10-05-reference-adversary/verification/definition_conformance.json create mode 100644 reference_audit/2026-10-05-reference-adversary/verification/definition_conformance.md create mode 100644 reference_audit/2026-10-05-reference-adversary/verification/flags_table.md create mode 100644 reference_audit/2026-10-05-reference-adversary/verification/publication_sources.json create mode 100644 reference_audit/2026-10-05-reference-adversary/verification/publication_sources.md create mode 100755 scripts/run_reference_adversary_claude.sh create mode 100755 scripts/run_reference_adversary_codex.sh create mode 100644 tests/test_consensus.py create mode 100644 tests/test_definition_conformance.py create mode 100644 tests/test_publication_sources.py create mode 100644 tests/test_reference_adversary.py create mode 100644 tests/test_reference_adversary_runner.py diff --git a/docs/audit.md b/docs/audit.md index 7f16906b..0a1ce6ed 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -65,3 +65,92 @@ genuine model errors (`llm_error`) should survive to a scored snapshot — Re-running the script is safe: a case is skipped once it has a verdict, so interrupted runs resume and failures can be re-attempted by re-invoking. + +# Reference adversary + +The diagnosis judge above is built to explain misses, not to doubt references: +its prompt says to treat the reference and its derivation as correct, labels +engine facts authoritative, and allows doubt only through `reference_suspect` +with a concrete contradiction. On 2026-10-05 that design missed reference issues +a reading of the data found: Louisiana's 2026 standard deduction was +PolicyEngine's own CPI computation rather than a published amount; the +`payroll_tax` reference counted paid-leave employee shares that the employer +may, but need not, pass on, though the output asks for "mandatory employee state +payroll taxes"; and the income tax references left out a dependent's own return. +The reference adversary is a separate pass whose only job is to attack the +reference. Its verdicts never change a score. + +## Pipeline + +```bash +# 1. Consensus trigger: scored cells where a cluster of models shares a wrong +# answer (>= 15 of the models, or >= 3 of the top 5). +uv run policybench consensus-flags \ + --payload paper/snapshot/20260501/runs//data.json.gz \ + --output /consensus_flags.json + +# 2. One two-stage case per flagged cell, minus cells another audit owns. +uv run policybench adversary-prepare \ + --payload paper/snapshot/20260501/runs//data.json.gz \ + --flags /consensus_flags.json \ + --annotations-dir annotations/ \ + --skip-cells /covered_elsewhere.json \ + --adversary-dir + +# 3. Judge inside a Subfleet lane (subscription billing, never an API key). +scripts/run_reference_adversary_claude.sh # Claude lane +scripts/run_reference_adversary_codex.sh # Codex lane + +# 4. Verdict table and adjudication queue (one or more judges). +uv run policybench adversary-collect \ + --adversary-dir claude= --adversary-dir codex= \ + --output-dir +``` + +`consensus-flags` records its parameters in the report. `--prototype` +reproduces the 2026-10-05 prototype (answers truncated to whole dollars, +eligibility outputs never flagged): 41 of the frozen run's 1,928 scored cells. +The defaults round answers to the nearest dollar, which merges answers such as +13,387.65 and 13,388, and compare eligibility outputs by mismatch: 61 cells. + +## Two stages + +- **Stage 1, law first.** The judge sees the household prompt, the output's + definition from `benchmark_specs.json`, the consensus answer with each + member's model id, answer and explanation, and the reference value. It does + not see the engine derivation. It works the answer from primary law + (statutes, regulations, agency publications, forms and instructions), cites + each rule with its publication date and whether it predates the 2026-07-03 + reference freeze, and says which answer the law supports. The Claude runner + gives it only web tools (no file access) and blocks PolicyBench and + PolicyEngine sites; the Codex runner works from an empty directory and + rejects a stage 1 whose log shows a read of derivation-bearing paths. +- **Stage 2, reconcile.** A fresh call gets the frozen stage-1 JSON (bound by + its sha256) and only now the engine derivation. It returns a verdict: + `reference_holds`, `reference_wrong`, `definition_mismatch` or + `prompt_ambiguous`, with citations and a suggested adjudication. + +## Where verdicts go + +`adversary-collect` writes each judge's verdict table and +`adversary_adjudication_queue.csv`: every case with a verdict other than +`reference_holds`, in the case-notes schema with `reference_suspect=true`. +`policybench.reference_adversary.apply_adversary_flags` sets those flags on a +release's case notes, so `scripts/freeze_snapshot.py` refuses to freeze until +each carries a developer `reference_verdict` (`policybench.adjudications`): +the existing adjudication path. A change to a published reference or an +exclusion still needs a release and the maintainer's ruling. + +## Definition and publication checks + +Two engine-side checks run with the reference venv (policyengine-us 2.15.17 +plus the reference conventions) and write under +`reference_audit/2026-10-05-reference-adversary/verification/`: + +- `definition_conformance` lists the policyengine-us variables each output's + reference sums and tests them against the qualifiers in the output's + definition ("employee-side", "mandatory", "household", "excluding the ACA + PTC", "excluding local income and payroll taxes"). +- `publication_sources` lists every 2026 parameter value a scored reference + reads and reports whether its metadata cites a government publication dated + before the freeze, flagging computed or indexed values. diff --git a/policybench/audit.py b/policybench/audit.py index 0c947b1d..9a77f25f 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -281,13 +281,8 @@ def _scenario_questions( parametric knowledge with no tools; PolicyEngine's microsimulation is the \ reference. -The reference value and its derivation are generated directly from the \ -engine's computation trace, and the reference pipeline has survived an \ -adversarial review program: every wrong-reference hypothesis raised by \ -earlier audits was adjudicated against primary sources, and the few real \ -bugs found were fixed before this run. Treat the reference and its \ -derivation as correct. Your job is NOT to re-litigate the reference — it is \ -to explain each model's mistake decisively. +Treat the reference and its derivation as correct. Your job is NOT to \ +re-litigate the reference — it is to explain each model's mistake decisively. For every wrong model, write a `diagnosis`: 1-3 definitive sentences naming \ the exact rule, eligibility pathway, deduction, threshold, or computation \ diff --git a/policybench/cli.py b/policybench/cli.py index 8e894490..a1d59eca 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -582,6 +582,78 @@ def main(): "(default: refuse and list the hedged cases for re-judging)", ) + # Reference adversary: consensus trigger -> law-first judge -> adjudication + consensus_parser = subparsers.add_parser( + "consensus-flags", + help="Flag scored cells where a cluster of models shares a wrong answer", + ) + consensus_parser.add_argument( + "--payload", + required=True, + help="Frozen run payload (data.json.gz) or a release dashboard-data.json", + ) + consensus_parser.add_argument("--output", required=True, help="Report JSON path") + consensus_parser.add_argument( + "--prototype", + action="store_true", + help="Use the 2026-10-05 prototype's parameters (answers truncated to " + "whole dollars, eligibility outputs never flagged); explicit flags " + "still override", + ) + consensus_parser.add_argument("--min-models", type=int, default=None) + consensus_parser.add_argument("--top-k", type=int, default=None) + consensus_parser.add_argument("--min-top", type=int, default=None) + consensus_parser.add_argument("--tolerance", type=float, default=None) + consensus_parser.add_argument( + "--answer-rounding", + choices=["nearest", "truncate", "cents"], + default=None, + ) + consensus_parser.add_argument("--zero-cluster-min-models", type=int, default=None) + consensus_parser.add_argument( + "--binary-outputs", + choices=["mismatch", "skip"], + default=None, + help="Eligibility outputs: wrong when the flag differs (mismatch, the " + "default) or judged by the dollar tolerance and so never flagged (skip)", + ) + + adversary_prepare_parser = subparsers.add_parser( + "adversary-prepare", + help="Write two-stage reference-adversary prompts for consensus-flagged cells", + ) + adversary_prepare_parser.add_argument("--payload", required=True) + adversary_prepare_parser.add_argument( + "--flags", required=True, help="consensus-flags report JSON" + ) + adversary_prepare_parser.add_argument("--adversary-dir", required=True) + adversary_prepare_parser.add_argument( + "--annotations-dir", + default=None, + help="Frozen annotations holding us_case_reference_explanations.csv " + "(the stage-2 derivations); default: the payload's referenceExplanation", + ) + adversary_prepare_parser.add_argument( + "--skip-cells", + default=None, + help="JSON list of {scenario_id, variable, covered_by} cells another " + "audit already covers; they get no adversary case", + ) + + adversary_collect_parser = subparsers.add_parser( + "adversary-collect", + help="Fold reference-adversary verdicts into a verdict table and an " + "adjudication queue (changes no score)", + ) + adversary_collect_parser.add_argument( + "--adversary-dir", + action="append", + required=True, + metavar="LABEL=DIR", + help="One judge's adversary directory, optionally labeled (repeatable)", + ) + adversary_collect_parser.add_argument("--output-dir", required=True) + # Population weights weight_parser = subparsers.add_parser( "population-weights", @@ -1335,6 +1407,110 @@ def main(): f"{n_suspect} cases flag the PolicyEngine reference as suspect." ) + elif args.command == "consensus-flags": + from dataclasses import replace + + from policybench.consensus import ( + PROTOTYPE_PARAMS, + ConsensusParams, + consensus_report, + file_sha256, + load_us_payload, + ) + + params = PROTOTYPE_PARAMS if args.prototype else ConsensusParams() + overrides = { + name: getattr(args, name) + for name in ( + "min_models", + "top_k", + "min_top", + "tolerance", + "answer_rounding", + "zero_cluster_min_models", + "binary_outputs", + ) + if getattr(args, name) is not None + } + params = replace(params, **overrides) + payload_path = Path(args.payload) + report = consensus_report( + load_us_payload(payload_path), + params, + source=str(payload_path), + source_sha256=file_sha256(payload_path), + ) + _ensure_parent_dir(args.output) + Path(args.output).write_text(json.dumps(report, indent=2) + "\n") + print( + f"Flagged {report['flagged_cells']} of {report['scored_cells']} scored " + f"cells ({report['models']} models) -> {args.output}" + ) + + elif args.command == "adversary-prepare": + from policybench.consensus import load_us_payload + from policybench.reference_adversary import ( + build_adversary_cases, + load_derivations, + prepare_adversary, + ) + + flags = json.loads(Path(args.flags).read_text())["flags"] + skipped: dict[tuple[str, str], str] = {} + if args.skip_cells: + for cell in json.loads(Path(args.skip_cells).read_text()): + key = (str(cell["scenario_id"]), str(cell["variable"])) + skipped[key] = str(cell.get("covered_by", "")) + kept = [f for f in flags if (f["scenario_id"], f["variable"]) not in skipped] + derivations = ( + load_derivations(Path(args.annotations_dir)) + if args.annotations_dir + else None + ) + cases = build_adversary_cases( + load_us_payload(Path(args.payload)), kept, derivations=derivations + ) + prepare_adversary(Path(args.adversary_dir), cases) + print( + f"Prepared {len(cases)} adversary cases under {args.adversary_dir} " + f"({len(flags) - len(kept)} flagged cells skipped as covered elsewhere). " + "Run scripts/run_reference_adversary_claude.sh or " + "scripts/run_reference_adversary_codex.sh inside a Subfleet lane." + ) + + elif args.command == "adversary-collect": + from policybench.reference_adversary import ( + adjudication_queue, + collect_adversary, + merge_judges, + ) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + by_judge = {} + for spec in args.adversary_dir: + label, _, directory = spec.rpartition("=") + label = label or Path(directory).name + out = collect_adversary(Path(directory)) + by_judge[label] = out["verdicts"] + for key in ("verdicts", "missing", "inconsistent"): + out[key].to_csv( + output_dir / f"adversary_{label}_{key}.csv", index=False + ) + print( + f"{label}: {len(out['verdicts'])} verdicts, " + f"{len(out['missing'])} missing, " + f"{len(out['inconsistent'])} inconsistent" + ) + merged = merge_judges(by_judge) + merged.to_csv(output_dir / "adversary_merged_verdicts.csv", index=False) + queue = adjudication_queue(merged) + queue.to_csv(output_dir / "adversary_adjudication_queue.csv", index=False) + print( + f"{len(queue)} of {len(merged)} cases go to developer adjudication " + f"(verdict other than reference_holds) -> {output_dir}" + ) + elif args.command == "population-weights": from policybench.config import TAX_YEAR from policybench.population_weights import write_population_weight_payload diff --git a/policybench/consensus.py b/policybench/consensus.py new file mode 100644 index 00000000..0dc728f9 --- /dev/null +++ b/policybench/consensus.py @@ -0,0 +1,365 @@ +"""Flag scored cells where a cluster of models agrees on the same wrong answer. + +A cell is one scenario's one output. Models answer from parametric knowledge, +so a single model missing the reference says little about the reference. Many +models landing on the same wrong number, or several of the strongest models +doing so, is a different signal: it suggests the reference itself may be wrong +(a convention the prompt did not state, an engine defect, or later law). This +module is the trigger that picks those cells out of a frozen dashboard payload +so a reviewer or a reference adversary can look at them first. It never +changes a score or a reference; it only lists cells. + +The rule, over scored cells only: + +1. Each model's parsed prediction is rounded to a key (``answer_rounding``): + ``"nearest"`` is half-up to whole dollars (``math.floor(x + 0.5)``), + ``"truncate"`` drops the fraction toward zero (``int(x)``), and ``"cents"`` + is half-up to 0.01 on the prediction's shortest decimal form. Models that + share a key form a cluster. A prediction that did not parse, or that is + ``None``, NaN or infinite, joins no cluster. +2. A cluster is wrong when its key differs from the reference by more than + ``tolerance`` dollars. On an eligibility output (``metric_type`` binary in + ``benchmark_specs.json``) a 0/1 answer can never differ from a 0/1 + reference by more than a dollar, so ``binary_outputs`` decides those: + ``"mismatch"`` (the default) calls a cluster wrong when its key is not the + reference, and ``"skip"`` applies the dollar tolerance and so never flags + one, as the prototype did. The models-exact count follows the same test. +3. A wrong cluster triggers when it has at least ``min_models`` members + (``"min_models"``) or at least ``min_top`` of the ``top_k`` best models by + the payload's ``modelStats`` order (``"min_top"``). +4. A wrong cluster whose key is zero also needs at least + ``zero_cluster_min_models`` members, so a few strong models agreeing that + a household gets nothing cannot flag a cell on the ``min_top`` trigger + alone. On the frozen 20260612 payload this guard removes no flag at the + prototype, default or cents parameters; it bounds what the trigger can do + on other boards. + +A cell is flagged when any of its wrong clusters triggers, and the flag lists +only the clusters that triggered. ``PROTOTYPE_PARAMS`` reproduces the +prototype that first ran this rule on 2026-10-05: answers truncated to whole +dollars and eligibility outputs never flagged (41 of the frozen run's 1,928 +scored cells). The default parameters round to the nearest dollar, which also +merges answers such as 13,387.65 and 13,388 that truncation splits (43 amount +cells), and compare eligibility flags by mismatch (18 more). +""" + +from __future__ import annotations + +import gzip +import hashlib +import json +import math +from dataclasses import asdict, dataclass +from decimal import ROUND_HALF_UP, Decimal +from pathlib import Path +from typing import Any + +ANSWER_ROUNDINGS = ("nearest", "truncate", "cents") +BINARY_OUTPUTS = ("mismatch", "skip") +TRIGGERS = ("min_models", "min_top") +# Rounded keys are kept as integers in these units so that clusters group by +# exact equality rather than by float comparison. +_UNITS_PER_DOLLAR = {"nearest": 1, "truncate": 1, "cents": 100} +_CENT = Decimal("0.01") + + +def _is_count(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +@dataclass(frozen=True) +class ConsensusParams: + """Thresholds of the consensus trigger; see the module docstring.""" + + min_models: int = 15 + top_k: int = 5 + min_top: int = 3 + tolerance: float = 1.0 + answer_rounding: str = "nearest" + zero_cluster_min_models: int = 15 + binary_outputs: str = "mismatch" + + def __post_init__(self) -> None: + for name in ("min_models", "top_k", "min_top", "zero_cluster_min_models"): + value = getattr(self, name) + if not _is_count(value) or value < 1: + raise ValueError(f"{name} must be an integer of at least 1: {value!r}") + if self.min_top > self.top_k: + raise ValueError( + f"min_top ({self.min_top}) cannot exceed top_k ({self.top_k}); " + "the min_top trigger could never fire" + ) + tolerance = self.tolerance + if ( + isinstance(tolerance, bool) + or not isinstance(tolerance, (int, float)) + or not math.isfinite(tolerance) + or tolerance < 0 + ): + raise ValueError( + f"tolerance must be a finite number of at least 0: {tolerance!r}" + ) + object.__setattr__(self, "tolerance", float(tolerance)) + if self.answer_rounding not in ANSWER_ROUNDINGS: + raise ValueError( + f"answer_rounding must be one of {ANSWER_ROUNDINGS}: " + f"{self.answer_rounding!r}" + ) + if self.binary_outputs not in BINARY_OUTPUTS: + raise ValueError( + f"binary_outputs must be one of {BINARY_OUTPUTS}: " + f"{self.binary_outputs!r}" + ) + + def to_dict(self) -> dict[str, Any]: + """The parameters as a JSON-ready dict, in field order.""" + return asdict(self) + + +PROTOTYPE_PARAMS = ConsensusParams(answer_rounding="truncate", binary_outputs="skip") + + +def load_us_payload(path: Path | str) -> dict: + """Read a US dashboard payload from ``.json`` or ``.json.gz``. + + Accepts both the per-country payload frozen under ``paper/snapshot`` and + the combined ``{"countries": {"us": ...}}`` shape of a dashboard release + asset, and returns the US payload either way. + """ + path = Path(path) + raw = path.read_bytes() + if raw[:2] == b"\x1f\x8b": + raw = gzip.decompress(raw) + payload = json.loads(raw.decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path}: payload is not a JSON object") + if "countries" in payload: + countries = payload["countries"] + if not isinstance(countries, dict) or "us" not in countries: + raise ValueError(f"{path}: combined payload has no 'us' country") + payload = countries["us"] + country = payload.get("country") + if country is not None and country != "us": + raise ValueError(f"{path}: payload is for country {country!r}, not 'us'") + if "scenarioPredictions" not in payload or "modelStats" not in payload: + raise ValueError(f"{path}: payload lacks scenarioPredictions or modelStats") + return payload + + +def file_sha256(path: Path | str) -> str: + """The sha256 of a file's bytes, as lowercase hex.""" + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def ranked_models(payload: dict) -> list[str]: + """Model ids in the payload's ``modelStats`` order (best first). + + The dashboard export orders ``modelStats`` by the headline metric, so this + order is the only ranking the trigger uses; the order of models inside + ``scenarioPredictions`` never matters. + """ + stats = payload.get("modelStats") + if not isinstance(stats, list): + raise ValueError("payload modelStats must be a list") + models = [row["model"] for row in stats] + duplicates = sorted({model for model in models if models.count(model) > 1}) + if duplicates: + raise ValueError(f"modelStats lists models more than once: {duplicates}") + return models + + +def _answer_units(value: float, rounding: str) -> int: + """A prediction's rounded key in integer units of the rounding mode.""" + if rounding == "nearest": + return math.floor(value + 0.5) + if rounding == "truncate": + return int(value) + cents = Decimal(repr(value)).quantize(_CENT, rounding=ROUND_HALF_UP) + return int(cents * 100) + + +def _answer(entry: Any) -> float | None: + """A model's prediction if it parsed to a finite number, else ``None``.""" + if not isinstance(entry, dict) or entry.get("parsed") is False: + return None + value = entry.get("prediction") + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + value = float(value) + return value if math.isfinite(value) else None + + +def _is_binary(variable: str) -> bool: + """Whether an output is an eligibility flag (``metric_type`` binary).""" + from policybench.spec import metric_type_for_output + + return metric_type_for_output(variable) == "binary" + + +def _misses(answer: float, reference: float, binary: bool, tolerance: float) -> bool: + """Whether an answer (or a cluster's key) misses the reference.""" + if binary: + return answer != reference + return abs(answer - reference) > tolerance + + +def _cell_reference( + scenario_id: str, variable: str, cell: dict[str, Any] +) -> tuple[bool, float | None]: + """Whether a cell is scored, and its reference value if it is.""" + scored = {bool(entry.get("scored")) for entry in cell.values()} + if len(scored) != 1: + raise ValueError(f"{scenario_id} {variable}: models disagree on scored") + if not scored.pop(): + return False, None + references = {json.dumps(entry.get("groundTruth")) for entry in cell.values()} + if len(references) != 1: + raise ValueError(f"{scenario_id} {variable}: models disagree on groundTruth") + reference = next(iter(cell.values())).get("groundTruth") + if ( + isinstance(reference, bool) + or not isinstance(reference, (int, float)) + or not math.isfinite(reference) + ): + raise ValueError( + f"{scenario_id} {variable}: scored cell has no finite reference: " + f"{reference!r}" + ) + return True, float(reference) + + +def _scan(payload: dict, params: ConsensusParams) -> tuple[int, list[dict]]: + """Count the scored cells and collect the flags, in (scenario, variable).""" + top_models = ranked_models(payload)[: params.top_k] + top_rank = {model: rank for rank, model in enumerate(top_models)} + scale = _UNITS_PER_DOLLAR[params.answer_rounding] + scenarios = payload.get("scenarios") or {} + predictions = payload["scenarioPredictions"] + scored_cells = 0 + flags: list[dict] = [] + for scenario_id in sorted(predictions): + outputs = predictions[scenario_id] + for variable in sorted(outputs): + cell = outputs[variable] + if not cell: + continue + scored, reference = _cell_reference(scenario_id, variable, cell) + if not scored: + continue + scored_cells += 1 + binary = params.binary_outputs == "mismatch" and _is_binary(variable) + answers = { + model: answer + for model, entry in sorted(cell.items()) + if (answer := _answer(entry)) is not None + } + keys = { + model: _answer_units(answer, params.answer_rounding) + for model, answer in answers.items() + } + groups: dict[int, list[str]] = {} + for model, units in keys.items(): + groups.setdefault(units, []).append(model) + # An eligibility answer counts as exact by its rounded value; an + # amount by its raw value within the tolerance. + compared = ( + {model: units / scale for model, units in keys.items()} + if binary + else answers + ) + clusters = [] + triggers: set[str] = set() + for units, members in groups.items(): + answer = units / scale + if not _misses(answer, reference, binary, params.tolerance): + continue + top = sorted( + (model for model in members if model in top_rank), + key=top_rank.__getitem__, + ) + met = [] + if len(members) >= params.min_models: + met.append("min_models") + if len(top) >= params.min_top: + met.append("min_top") + if not met: + continue + if units == 0 and len(members) < params.zero_cluster_min_models: + continue + triggers.update(met) + clusters.append( + { + "answer": float(answer), + "n_models": len(members), + "n_top": len(top), + "models": sorted(members), + "top_models": top, + "predictions": { + model: answers[model] for model in sorted(members) + }, + } + ) + if not clusters: + continue + if scenario_id not in scenarios: + raise ValueError(f"{scenario_id}: missing from payload scenarios") + clusters.sort(key=lambda cluster: (-cluster["n_models"], cluster["answer"])) + flags.append( + { + "scenario_id": scenario_id, + "variable": variable, + "state": scenarios[scenario_id].get("state"), + "reference": reference, + "models_answered": len(answers), + "models_exact": sum( + not _misses(value, reference, binary, params.tolerance) + for value in compared.values() + ), + "trigger": sorted(triggers), + "clusters": clusters, + } + ) + return scored_cells, flags + + +def consensus_flags( + payload: dict, params: ConsensusParams = ConsensusParams() +) -> list[dict]: + """The scored cells where a wrong-answer cluster triggers. + + Each flag carries the cell's scenario, variable, state and reference, how + many models answered and how many matched the reference (within the + tolerance; an eligibility flag by its rounded value), the triggers that + fired, and the triggering clusters (largest first, then by answer). Flags + are sorted by scenario id, then variable. + """ + return _scan(payload, params)[1] + + +def consensus_report( + payload: dict, + params: ConsensusParams = ConsensusParams(), + *, + source: str = "", + source_sha256: str = "", +) -> dict: + """The flags with the parameters and counts needed to reproduce them. + + ``source`` and ``source_sha256`` name the payload file the caller read, so + a saved report stays bound to the exact bytes it was computed from. + """ + scored_cells, flags = _scan(payload, params) + return { + "params": params.to_dict(), + "top_models": ranked_models(payload)[: params.top_k], + "source": source, + "source_sha256": source_sha256, + "models": len(ranked_models(payload)), + "scored_cells": scored_cells, + "flagged_cells": len(flags), + "flags": flags, + } diff --git a/policybench/definition_conformance.py b/policybench/definition_conformance.py new file mode 100644 index 00000000..f3908468 --- /dev/null +++ b/policybench/definition_conformance.py @@ -0,0 +1,1578 @@ +"""Check each benchmark output's reference against the qualifiers in its definition. + +Every PolicyBench output has a definition, the ``prompt`` text of its entry in +``benchmark_specs.json`` (the text the models saw), and a reference, the +policyengine-us variable named by its ``pe_variable``. That variable is +usually a sum of other variables. policyengine-core 3.32.8 sums the variables +a variable lists in ``adds`` and subtracts those in ``subtracts`` only when no +formula applies at the period, and either attribute may instead be a parameter +path whose value at the period's start is the list +(``Simulation._run_formula``). Some sums are written as formulas instead, such +as ``employee_payroll_tax``'s ``add(tax_unit, period, COMPONENTS)``; +``parse_formula_sum`` reads a formula's source and keeps it only when it is a +plain signed sum of other variables. + +``component_tree`` walks that sum from the output's variable to its leaves, +carrying each component's path and its sign relative to the output. A leaf is +a variable the walk does not expand: an input, a formula that is not a plain +sum, a name that is not a variable, or a repeat of a variable already on the +path (a cycle). ``QUALIFIER_RULES`` are tests of what a definition says: each +rule has a pattern that detects a qualifier in the definition ("employee-side", +"excluding local income and payroll taxes", "before refundable credits", ...) +and a test that says whether one component breaks it. ``conformance_report`` +applies the detected rules to every component, attaches the scored cells where +a breaking component is nonzero and reaches the reference (an ancestor with +``defined_for`` can zero it), lists the material leaves the definition +neither names nor excludes (``definition_silent``, information only), checks +that every expanded node equals the signed sum of its children in every cell, +and evaluates household scope from per-member records an engine run supplies. + +Everything here is a pure function of an abstract variable graph +(``VariableInfo``) except ``variable_graph_from_system``, which reads a graph +from a policyengine-core tax-benefit system object without importing +policyengine. The engine-side driver is +``reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py``. +""" + +from __future__ import annotations + +import ast +import inspect +import re +import textwrap +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +SCHEMA_VERSION = 1 +DEFAULT_INSTANT = "2026-01-01" +# The frozen prompts list every output under this heading, so every output is +# asked for the household as a whole. +HOUSEHOLD_PROMPT_PHRASE = "Provide the following policy quantities for this household" +# Dollar amounts at or below this are treated as zero. +TOLERANCE = 0.005 + +ADDS = "adds" +SUBTRACTS = "subtracts" +FORMULA_ADDS = "formula_adds" +FORMULA_SUBTRACTS = "formula_subtracts" +EDGE_SIGNS = {ADDS: 1, SUBTRACTS: -1, FORMULA_ADDS: 1, FORMULA_SUBTRACTS: -1} + +OPTIONAL_PASS_THROUGH = "optional_employer_pass_through" + + +@dataclass(frozen=True) +class VariableInfo: + """One variable of the graph, as the sum walk needs it. + + ``adds`` and ``subtracts`` hold what policyengine-core sums when no + formula applies. When the variable declared one of them as a parameter + path, the path is in ``adds_parameter`` or ``subtracts_parameter`` and the + tuple holds the parameter's value at the graph's instant. + ``formula_adds`` and ``formula_subtracts`` hold the terms of a formula + that is a plain signed sum (see ``parse_formula_sum``); they are empty for + any other formula. ``parameter_text`` holds the descriptions of the + parameters the formula reads, as further text evidence for the rules. + ``defined_for`` names the variable whose positive values mark where the + engine computes this one; elsewhere it takes its default (0), whatever its + parts sum to. + """ + + name: str + entity: str + label: str = "" + documentation: str = "" + adds: tuple[str, ...] = () + subtracts: tuple[str, ...] = () + adds_parameter: str | None = None + subtracts_parameter: str | None = None + has_formula: bool = False + definition_period: str = "year" + formula_adds: tuple[str, ...] = () + formula_subtracts: tuple[str, ...] = () + parameter_text: str = "" + defined_for: str | None = None + + def edges(self) -> tuple[tuple[str, str], ...]: + """The ``(edge, child)`` pairs the engine sums for this variable, in order. + + A formula takes precedence over ``adds`` and ``subtracts`` at run time, + so a variable with a formula expands only through the formula's terms. + """ + if self.has_formula: + return tuple((FORMULA_ADDS, child) for child in self.formula_adds) + tuple( + (FORMULA_SUBTRACTS, child) for child in self.formula_subtracts + ) + return tuple((ADDS, child) for child in self.adds) + tuple( + (SUBTRACTS, child) for child in self.subtracts + ) + + +def _leaf_reason(info: VariableInfo | None) -> str | None: + if info is None: + return "not_a_variable" + if info.edges(): + return None + if info.has_formula: + if info.adds or info.subtracts: + return "formula_overrides_adds" + return "formula" + return "input" + + +def component_tree(graph: Mapping[str, VariableInfo], root: str) -> list[dict]: + """Every node of ``root``'s sum, depth first in declaration order. + + Each node carries its ``path`` from the root, its ``sign`` relative to the + root (the product of the edge signs along the path), the ``edge`` that + reached it, the parameter path that edge came from (``via_parameter``), + its entity and label, and ``kind``: ``root``, ``intermediate`` or + ``leaf``. Leaves carry a ``leaf_reason``: ``input``, ``formula`` (a + formula that is not a plain sum), ``formula_overrides_adds``, + ``not_a_variable`` or ``cycle`` (the variable is already on the path, so + the walk stops there). + """ + nodes: list[dict] = [] + stack: list[tuple[str, tuple[str, ...], int, str | None, str | None]] = [ + (root, (), 1, None, None) + ] + while stack: + name, ancestors, sign, edge, via_parameter = stack.pop() + path = ancestors + (name,) + info = graph.get(name) + cycle = name in ancestors + reason = "cycle" if cycle else _leaf_reason(info) + if not ancestors: + kind = "root" if reason is None else "leaf" + else: + kind = "intermediate" if reason is None else "leaf" + nodes.append( + { + "name": name, + "path": list(path), + "depth": len(ancestors), + "sign": sign, + "edge": edge, + "via_parameter": via_parameter, + "entity": info.entity if info is not None else None, + "label": info.label if info is not None else "", + "defined_for": info.defined_for if info is not None else None, + "kind": kind, + "leaf_reason": reason, + } + ) + if reason is not None: + continue + children = [] + for child_edge, child in info.edges(): + parameter = None + if child_edge == ADDS: + parameter = info.adds_parameter + elif child_edge == SUBTRACTS: + parameter = info.subtracts_parameter + children.append( + (child, path, sign * EDGE_SIGNS[child_edge], child_edge, parameter) + ) + stack.extend(reversed(children)) + return nodes + + +# ----- Qualifier rules ----- # + + +@dataclass(frozen=True) +class Component: + """What a qualifier rule sees of one node of an output's sum.""" + + name: str + label: str + documentation: str + entity: str | None + sign: int + path: tuple[str, ...] + kind: str + parameter_text: str = "" + law_classification: Mapping[str, Any] | None = None + # Names subtracted by any ancestor of this node (its parent included). + subtracted_along_path: tuple[str, ...] = () + + @property + def text(self) -> str: + """Label, documentation and parameter descriptions, one sentence apart.""" + parts = (self.label, self.documentation, self.parameter_text) + return " ".join( + part.strip() if part.strip().endswith(".") else f"{part.strip()}." + for part in parts + if part.strip() + ) + + +@dataclass(frozen=True) +class RuleHit: + """A component that breaks a qualifier, with the reason and its evidence. + + ``evidence`` is ``variable_name`` (the variable's name or label), + ``engine_text`` (its documentation or the descriptions of parameters its + formula reads), ``law_classification`` (a cited classification supplied + by the caller) or ``engine_text+law_classification``. + """ + + reason: str + evidence: str + + +@dataclass(frozen=True) +class QualifierRule: + """One qualifier a definition can carry and the component test for it. + + ``pattern`` detects the qualifier in a definition (case-insensitive). + ``test`` reports a component that breaks it, or ``None``. A rule with + ``scope="household"`` has no component test; ``conformance_report`` + evaluates it from household-scope records instead. ``effect`` names the + variable whose cell value measures a hit's effect on the reference when + that is not the component itself. + """ + + id: str + description: str + pattern: str + test: Callable[[Component], RuleHit | None] | None = None + scope: str = "component" + effect: Callable[[Component], str | None] | None = None + + def detects(self, definition: str, *, household_prompt: bool = False) -> bool: + if self.scope == "household" and household_prompt: + return True + return re.search(self.pattern, definition, re.IGNORECASE) is not None + + +def _describe(component: Component) -> str: + if component.label: + return f"'{component.name}' ({component.label})" + return f"'{component.name}'" + + +_EMPLOYER_NAME = re.compile(r"(?:^|_)(?:employer|futa)(?:_|$)") +_EMPLOYER_LABEL = re.compile( + r"\bemployer\b|\bFUTA\b|federal\s+unemployment\s+tax", re.IGNORECASE +) +_EMPLOYEE_NAME = re.compile(r"(?:^|_)employee(?:_|$)") +_EMPLOYEE_LABEL = re.compile(r"\bemployee\b", re.IGNORECASE) + + +def _employer_side(component: Component) -> RuleHit | None: + employer = _EMPLOYER_NAME.search(component.name) or _EMPLOYER_LABEL.search( + component.label + ) + employee = _EMPLOYEE_NAME.search(component.name) or _EMPLOYEE_LABEL.search( + component.label + ) + if employer and not employee: + return RuleHit( + f"{_describe(component)} is an employer-side amount", "variable_name" + ) + return None + + +# Wording that says an amount rests on an employer's optional election rather +# than on a contribution the law requires the employee to pay. +OPTIONAL_ELECTION = re.compile( + r"assum\w*\s+(?:that\s+)?(?:the\s+|an\s+)?employers?\s+" + r"(?:withholds?|deducts?|collects?|elects?|passes)" + r"|maximum\s+permitted\s+employee\s+share" + r"|employers?\s+(?:may|can)\s+(?:choose\s+to\s+|elect\s+to\s+)?" + r"(?:deduct|withhold|collect|pass)" + r"|allowed,?\s+but\s+not\s+required" + r"|at\s+the\s+employer'?s\s+(?:option|discretion|election|choice)" + r"|\boptional\b|\bvoluntar\w*", + re.IGNORECASE, +) + + +def _sentence_around(text: str, match: re.Match[str]) -> str: + start = text.rfind(".", 0, match.start()) + 1 + end = text.find(".", match.end()) + end = len(text) if end < 0 else end + 1 + return " ".join(text[start:end].split()) + + +def _optional_election(component: Component) -> RuleHit | None: + text = component.text + match = OPTIONAL_ELECTION.search(text) + law = component.law_classification or {} + law_optional = law.get("classification") == OPTIONAL_PASS_THROUGH + reasons = [] + evidence = [] + if match: + reasons.append( + f"{_describe(component)} rests on an optional employer election: " + f'"{_sentence_around(text, match)}"' + ) + evidence.append("engine_text") + if law_optional: + citation = law.get("citation") or "no citation given" + reasons.append( + f"{_describe(component)} is classified as an optional employer " + f"pass-through ({citation})" + ) + evidence.append("law_classification") + if not reasons: + return None + return RuleHit("; ".join(reasons), "+".join(evidence)) + + +def _self_employment_tax(component: Component) -> RuleHit | None: + if re.search(r"self_employment\w*_tax|(?:^|_)se_tax(?:_|$)", component.name) or ( + re.search( + r"self[- ]employment\s+(?:social\s+security\s+|medicare\s+)?tax", + component.label, + re.IGNORECASE, + ) + ): + return RuleHit( + f"{_describe(component)} is self-employment tax", "variable_name" + ) + return None + + +def _additional_medicare_tax(component: Component) -> RuleHit | None: + if "additional_medicare" in component.name or re.search( + r"additional\s+medicare", component.label, re.IGNORECASE + ): + return RuleHit( + f"{_describe(component)} is Additional Medicare Tax", "variable_name" + ) + return None + + +def _employee_payroll_tax(component: Component) -> RuleHit | None: + if re.search(r"(?:^|_)employee(?:_|$)", component.name) and re.search( + r"tax|contribution", component.name + ): + return RuleHit( + f"{_describe(component)} is an employee payroll tax", "variable_name" + ) + return None + + +def _premium_tax_credit(component: Component) -> RuleHit | None: + if re.search( + r"premium_tax_credit|(?:^|_)aca_ptc(?:_|$)", component.name + ) or re.search(r"premium\s+tax\s+credit", component.label, re.IGNORECASE): + return RuleHit( + f"{_describe(component)} is the ACA Premium Tax Credit", "variable_name" + ) + return None + + +# Name fragments of local (city or county) income, wage and earnings taxes. +LOCAL_NAME = re.compile( + r"(?:^|_)local(?:_|$)|(?:^|_)nyc(?:_|$)|philadelphia|kansas_city|st_louis" + r"|wilmington|denver|yonkers|(?:^|_)county(?:_|$)" +) +LOCAL_LABEL = re.compile(r"\b(?:local|county|city|NYC)\b", re.IGNORECASE) + + +def _local_tax(component: Component) -> RuleHit | None: + if LOCAL_NAME.search(component.name) or LOCAL_LABEL.search(component.label): + return RuleHit( + f"{_describe(component)} is a local (city or county) amount", + "variable_name", + ) + return None + + +_NON_REFUNDABLE = re.compile(r"non_?refundable|non-?refundable", re.IGNORECASE) + + +def _is_refundable_credit(component: Component) -> bool: + name = component.name + label = component.label + if _NON_REFUNDABLE.search(name) or _NON_REFUNDABLE.search(label): + return False + return bool( + re.search(r"(?:^|_)refundable(?:_|$)", name) + or re.search(r"(?:^|_)eitc(?:_|$)", name) + or re.search(r"\brefundable\b", label, re.IGNORECASE) + or re.search(r"earned\s+income\s+(?:tax\s+)?credit", label, re.IGNORECASE) + ) + + +def _refundable_credit_subtracted(component: Component) -> RuleHit | None: + if component.sign < 0 and _is_refundable_credit(component): + return RuleHit( + f"{_describe(component)} is a refundable credit subtracted from an " + "output defined before refundable credits", + "variable_name", + ) + return None + + +_BEFORE_CREDITS_NAME = re.compile(r"before_(?:non_?refundable_)?credits") + + +def _before_nonrefundable_credits(component: Component) -> RuleHit | None: + # Names only: labels are looser. policyengine-us 2.15.17 labels + # va_income_tax_before_refundable_credits "Virginia income tax before + # credits" while its formula subtracts va_non_refundable_credits. + if not _BEFORE_CREDITS_NAME.search(component.name): + return None + if any(_NON_REFUNDABLE.search(name) for name in component.subtracted_along_path): + return None + return RuleHit( + f"{_describe(component)} is a tax before credits, and nothing on its path " + "subtracts nonrefundable credits", + "variable_name", + ) + + +def _companion_nonrefundable_credits(component: Component) -> str | None: + """The variable holding the credits a before-credits component leaves out. + + Follows the policyengine-us naming convention ``_non_refundable_ + credits`` (e.g. ``ms_non_refundable_credits`` for + ``ms_income_tax_before_credits_unit``); the engine run only uses it when + that variable exists. + """ + prefix = component.name.split("_", 1)[0] + return f"{prefix}_non_refundable_credits" + + +QUALIFIER_RULES: tuple[QualifierRule, ...] = ( + QualifierRule( + id="employee_side", + description="Employee-side output: no employer payroll tax, FUTA or " + "employer unemployment-insurance tax.", + pattern=r"\bemployee[- ]side\b|\bexclud\w*\s+employer\b", + test=_employer_side, + ), + QualifierRule( + id="mandatory", + description="Mandatory amounts only: no amount that rests on an " + "employer's optional election to pass a premium through to wages.", + pattern=r"\bmandatory\b", + test=_optional_election, + ), + QualifierRule( + id="excludes_self_employment_tax", + description="Self-employment tax is excluded.", + pattern=r"\bexclud\w*\b[^.]*?\bself[- ]employment\s+tax", + test=_self_employment_tax, + ), + QualifierRule( + id="excludes_additional_medicare_tax", + description="Additional Medicare Tax is excluded.", + pattern=r"\bexclud\w*\b[^.]*?\badditional\s+medicare\s+tax", + test=_additional_medicare_tax, + ), + QualifierRule( + id="excludes_employee_payroll_taxes", + description="Employee payroll taxes are excluded.", + pattern=r"\bexclud\w*\b[^.]*?\bemployee\s+payroll\s+tax", + test=_employee_payroll_tax, + ), + QualifierRule( + id="excludes_premium_tax_credit", + description="The ACA Premium Tax Credit is excluded.", + pattern=r"\bexclud\w*\b[^.]*?\bpremium\s+tax\s+credit", + test=_premium_tax_credit, + ), + QualifierRule( + id="excludes_local_tax", + description="Local income and payroll taxes are excluded.", + pattern=r"\bexclud\w*\b[^.]*?\blocal\b", + test=_local_tax, + ), + QualifierRule( + id="state_scope", + description="A state individual income tax amount: no city or county " + "income tax or credit.", + pattern=r"\bstate\s+individual\s+income\s+tax\b", + test=_local_tax, + ), + QualifierRule( + id="before_refundable_credits", + description="Before refundable credits: no refundable credit is subtracted.", + pattern=r"\bbefore\s+refundable\s+credits\b", + test=_refundable_credit_subtracted, + ), + QualifierRule( + id="after_nonrefundable_credits", + description="After nonrefundable credits: a tax computed before credits " + "needs its nonrefundable credits subtracted.", + pattern=r"\bafter\s+non-?refundable\s+credits\b", + test=_before_nonrefundable_credits, + effect=_companion_nonrefundable_credits, + ), + QualifierRule( + id="household_scope", + description="A household amount: every member's income the output " + "would count, including a dependent's own return.", + pattern=r"\bhousehold\b", + scope="household", + ), +) + + +# ----- Definition-silent components ----- # + +_STOPWORDS = frozenset( + """ + a an and annual amount after at before by credit credits each federal for from + in income individual liability non nonrefundable of on or per refundable side + state states tax taxes the to total unit us usd with person household employee + payroll contribution contributions + """.split() +) +_STATE_NAME_TOKENS = frozenset( + """ + alabama alaska arizona arkansas california colorado connecticut delaware district + columbia florida georgia hawaii idaho illinois indiana iowa kansas kentucky + louisiana maine maryland massachusetts michigan minnesota mississippi missouri + montana nebraska nevada new hampshire jersey mexico york north carolina dakota + ohio oklahoma oregon pennsylvania rhode island south tennessee texas utah vermont + virginia washington west wisconsin wyoming + """.split() +) + + +def _tokens(text: str) -> set[str]: + text = text.lower().replace("non-refundable", "nonrefundable") + text = text.replace("non refundable", "nonrefundable") + return set(re.findall(r"[a-z0-9]+", text.replace("_", " "))) + + +def _distinctive_tokens(label: str) -> set[str]: + return _tokens(label) - _STOPWORDS - _STATE_NAME_TOKENS + + +def _is_named(node: Mapping[str, Any], definition_tokens: set[str]) -> bool: + label = node.get("label") or node["name"].replace("_", " ") + distinctive = _distinctive_tokens(label) + return not distinctive or bool(distinctive & definition_tokens) + + +# ----- Formula reading ----- # + + +class _Unsupported(Exception): + """A formula construct outside the plain signed sums this module reads.""" + + +@dataclass(frozen=True) +class _Terms: + items: tuple[tuple[int, str], ...] + + +@dataclass(frozen=True) +class _Names: + items: tuple[str, ...] + + +@dataclass(frozen=True) +class _ParamRef: + path: str + + +class _FormulaReader: + def __init__( + self, + func: ast.FunctionDef, + resolve_name: Callable[[str], Any] | None, + resolve_parameter: Callable[[str], Any] | None, + ) -> None: + args = [arg.arg for arg in func.args.args] + self.entity_arg = args[0] if args else None + self.parameters_arg = args[2] if len(args) >= 3 else None + self.resolve_name = resolve_name + self.resolve_parameter = resolve_parameter + self.env: dict[str, Any] = {} + + def run(self, body: list[ast.stmt]) -> _Terms: + for index, statement in enumerate(body): + if isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Constant + ): + continue + if isinstance(statement, ast.Assign): + if len(statement.targets) != 1 or not isinstance( + statement.targets[0], ast.Name + ): + raise _Unsupported("assignment target") + self.env[statement.targets[0].id] = self.value(statement.value) + continue + if isinstance(statement, ast.If): + if not _returns_zero(statement.body): + raise _Unsupported("conditional") + return self.run(list(statement.orelse) + list(body[index + 1 :])) + if isinstance(statement, ast.Return): + if statement.value is None: + raise _Unsupported("empty return") + result = self.value(statement.value) + if not isinstance(result, _Terms) or not result.items: + raise _Unsupported("return value") + return result + raise _Unsupported(type(statement).__name__) + raise _Unsupported("no return") + + def names(self, value: Any) -> tuple[str, ...]: + if isinstance(value, _Names): + return value.items + if isinstance(value, _ParamRef) and self.resolve_parameter is not None: + resolved = self.resolve_parameter(value.path) + if _is_name_list(resolved): + return tuple(resolved) + raise _Unsupported("list of variables") + + def value(self, node: ast.expr) -> Any: + if isinstance(node, (ast.List, ast.Tuple)): + items = [] + for element in node.elts: + if not ( + isinstance(element, ast.Constant) and isinstance(element.value, str) + ): + raise _Unsupported("list element") + items.append(element.value) + return _Names(tuple(items)) + if isinstance(node, ast.Name): + if node.id in self.env: + return self.env[node.id] + if self.resolve_name is not None: + resolved = self.resolve_name(node.id) + if _is_name_list(resolved): + return _Names(tuple(resolved)) + raise _Unsupported(f"name {node.id}") + if isinstance(node, (ast.Attribute, ast.Subscript)): + path = _parameter_chain(node, self.parameters_arg, self.env) + if path is None: + raise _Unsupported("attribute") + return _ParamRef(path) + if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Add, ast.Sub)): + left = self.value(node.left) + right = self.value(node.right) + if not isinstance(left, _Terms) or not isinstance(right, _Terms): + raise _Unsupported("arithmetic") + sign = 1 if isinstance(node.op, ast.Add) else -1 + return _Terms( + left.items + tuple((sign * s, name) for s, name in right.items) + ) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + operand = self.value(node.operand) + if not isinstance(operand, _Terms): + raise _Unsupported("negation") + return _Terms(tuple((-s, name) for s, name in operand.items)) + if isinstance(node, ast.Call): + return self.call(node) + raise _Unsupported(type(node).__name__) + + def call(self, node: ast.Call) -> _Terms: + if node.keywords or not isinstance(node.func, ast.Name): + raise _Unsupported("call") + function = node.func.id + if function == "add" and len(node.args) == 3: + names = self.names(self.value(node.args[2])) + return _Terms(tuple((1, name) for name in names)) + first = node.args[0] if node.args else None + is_string = isinstance(first, ast.Constant) and isinstance(first.value, str) + if function == "sum_contained_tax_units" and is_string: + return _Terms(((1, first.value),)) + if function == self.entity_arg and is_string and len(node.args) == 2: + return _Terms(((1, first.value),)) + raise _Unsupported(f"call {function}") + + +def _is_name_list(value: Any) -> bool: + return isinstance(value, (list, tuple)) and all( + isinstance(item, str) for item in value + ) + + +def _returns_zero(body: list[ast.stmt]) -> bool: + return ( + len(body) == 1 + and isinstance(body[0], ast.Return) + and isinstance(body[0].value, ast.Constant) + and body[0].value.value == 0 + ) + + +def _parameter_chain( + node: ast.expr, parameters_arg: str | None, env: Mapping[str, Any] +) -> str | None: + """The dotted parameter path an expression such as ``p.rate`` reads.""" + parts: list[str] = [] + while True: + if isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + elif ( + isinstance(node, ast.Subscript) + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, str) + ): + parts.append(node.slice.value) + node = node.value + else: + break + parts.reverse() + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == (parameters_arg or "parameters") + ): + prefix: list[str] = [] + elif isinstance(node, ast.Name) and isinstance(env.get(node.id), _ParamRef): + prefix = env[node.id].path.split(".") + else: + return None + path = ".".join(prefix + parts) + return path or None + + +def _function_def(source: str) -> ast.FunctionDef | None: + try: + module = ast.parse(textwrap.dedent(source)) + except SyntaxError: + return None + for node in ast.walk(module): + if isinstance(node, ast.FunctionDef): + return node + return None + + +def parse_formula_sum( + source: str, + *, + resolve_name: Callable[[str], Any] | None = None, + resolve_parameter: Callable[[str], Any] | None = None, +) -> tuple[tuple[str, ...], tuple[str, ...]] | None: + """Read a formula that is a plain signed sum into ``(adds, subtracts)``. + + The forms read are ``add(entity, period, LIST)``, where ``LIST`` is a + literal list of names, a name bound to one (a module constant, resolved + through ``resolve_name``) or a parameter whose value is one (resolved + through ``resolve_parameter``); ``sum_contained_tax_units("name", ...)``; + ``entity("name", period)``; names bound to any of these; ``+``, ``-`` and + unary ``-`` over them; and a leading ``if ...: return 0`` guard, which is + skipped. Anything else (``max_``, ``where``, products, loops, other + branches) returns ``None``: the formula is then a leaf. A reading is a + claim about the formula's source only; the engine run checks it against + computed values in every cell. + """ + func = _function_def(source) + if func is None: + return None + reader = _FormulaReader(func, resolve_name, resolve_parameter) + try: + terms = reader.run(list(func.body)) + except _Unsupported: + return None + adds = tuple(name for sign, name in terms.items if sign > 0) + subtracts = tuple(name for sign, name in terms.items if sign < 0) + return adds, subtracts + + +class _ParameterPaths(ast.NodeVisitor): + def __init__(self, parameters_arg: str) -> None: + self.parameters_arg = parameters_arg + self.env: dict[str, _ParamRef] = {} + self.paths: set[str] = set() + + def visit_Assign(self, node: ast.Assign) -> None: + self.visit(node.value) + path = _parameter_chain(node.value, self.parameters_arg, self.env) + if path is not None and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + self.env[target.id] = _ParamRef(path) + + def _chain(self, node: ast.expr) -> None: + path = _parameter_chain(node, self.parameters_arg, self.env) + if path is None: + self.generic_visit(node) + else: + self.paths.add(path) + + def visit_Attribute(self, node: ast.Attribute) -> None: + self._chain(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: + self._chain(node) + + +def formula_parameter_paths(source: str) -> tuple[str, ...]: + """The parameter paths a formula reads, as written (``p.rate`` resolved).""" + func = _function_def(source) + if func is None: + return () + args = [arg.arg for arg in func.args.args] + visitor = _ParameterPaths(args[2] if len(args) >= 3 else "parameters") + for statement in func.body: + visitor.visit(statement) + return tuple(sorted(visitor.paths)) + + +# ----- Reading a graph from a policyengine-core system ----- # + + +def _parameter_node(parameters: Any, path: str) -> Any: + node = parameters + for part in path.split("."): + children = getattr(node, "children", None) + if isinstance(children, Mapping) and part in children: + node = children[part] + else: + node = getattr(node, part) + return node + + +def _parameter_value(parameters: Any, path: str, instant: str) -> Any: + return _parameter_node(parameters, path)(instant) + + +def _list_attribute( + system: Any, value: Any, instant: str +) -> tuple[tuple[str, ...], str | None]: + if value is None: + return (), None + if isinstance(value, str): + return tuple(_parameter_value(system.parameters, value, instant)), value + return tuple(value), None + + +def _parameter_descriptions(parameters: Any, paths: Iterable[str]) -> str: + texts: list[str] = [] + for path in paths: + parts = path.split(".") + # Use the deepest prefix of the path that names a parameter node. + for end in range(len(parts), 0, -1): + try: + node = _parameter_node(parameters, ".".join(parts[:end])) + except (AttributeError, KeyError, TypeError): + continue + for text in ( + getattr(node, "description", None), + (getattr(node, "metadata", None) or {}).get("label"), + ): + if isinstance(text, str) and text.strip(): + text = " ".join(text.split()) + if text not in texts: + texts.append(text) + break + return " ".join(texts) + + +def _closure_values(function: Any) -> dict[str, Any]: + names = getattr(getattr(function, "__code__", None), "co_freevars", ()) + cells = getattr(function, "__closure__", None) or () + values = {} + for name, cell in zip(names, cells, strict=False): + try: + values[name] = cell.cell_contents + except ValueError: + continue + return values + + +def _read_formula( + system: Any, formula: Any, instant: str +) -> tuple[tuple[tuple[str, ...], tuple[str, ...]] | None, tuple[str, ...]]: + closure = _closure_values(formula) + if getattr(formula, "__name__", "") == "sum_of_variables" and ( + "variables" in closure + ): + names, _ = _list_attribute(system, closure["variables"], instant) + return (names, ()), () + try: + source = inspect.getsource(formula) + except (OSError, TypeError): + return None, () + module_globals = getattr(formula, "__globals__", {}) + + def resolve_name(name: str) -> Any: + if name in closure: + return closure[name] + return module_globals.get(name) + + def resolve_parameter(path: str) -> Any: + try: + return _parameter_value(system.parameters, path, instant) + except (AttributeError, KeyError, TypeError, ValueError): + return None + + parsed = parse_formula_sum( + source, resolve_name=resolve_name, resolve_parameter=resolve_parameter + ) + return parsed, formula_parameter_paths(source) + + +def _defined_for_name(value: Any) -> str | None: + if value is None: + return None + return str(getattr(value, "name", value)) + + +def _variable_info(system: Any, variable: Any, instant: str) -> VariableInfo: + adds, adds_parameter = _list_attribute(system, variable.adds, instant) + subtracts, subtracts_parameter = _list_attribute( + system, variable.subtracts, instant + ) + formula = variable.get_formula(instant) + formula_adds: tuple[str, ...] = () + formula_subtracts: tuple[str, ...] = () + parameter_text = "" + if formula is not None: + parsed, paths = _read_formula(system, formula, instant) + if parsed is not None: + formula_adds, formula_subtracts = parsed + parameter_text = _parameter_descriptions(system.parameters, paths) + period = getattr(variable, "definition_period", "year") + return VariableInfo( + name=variable.name, + entity=variable.entity.key, + label=" ".join(str(variable.label or "").split()), + documentation=" ".join(str(variable.documentation or "").split()), + adds=adds, + subtracts=subtracts, + adds_parameter=adds_parameter, + subtracts_parameter=subtracts_parameter, + has_formula=formula is not None, + definition_period=str(getattr(period, "value", period)), + formula_adds=formula_adds, + formula_subtracts=formula_subtracts, + parameter_text=parameter_text, + defined_for=_defined_for_name(getattr(variable, "defined_for", None)), + ) + + +def variable_graph_from_system( + system: Any, + instant: str = DEFAULT_INSTANT, + *, + roots: Iterable[str] | None = None, +) -> dict[str, VariableInfo]: + """Read the variable graph of a policyengine-core tax-benefit system. + + ``adds`` or ``subtracts`` given as a parameter path are resolved to the + parameter's list at ``instant``, as policyengine-core does at a period's + start. A variable has a formula when ``get_formula(instant)`` returns one; + its source is then read with ``parse_formula_sum``. With ``roots``, only + the variables their sums reach are read; otherwise every variable is. + """ + variables = system.variables + queue = list(variables) if roots is None else list(roots) + graph: dict[str, VariableInfo] = {} + while queue: + name = queue.pop(0) + if name in graph or name not in variables: + continue + info = _variable_info(system, variables[name], instant) + graph[name] = info + if roots is not None: + queue.extend(child for _, child in info.edges()) + return graph + + +# ----- The report ----- # + + +def _spec_field(spec: Any, name: str) -> Any: + if isinstance(spec, Mapping): + return spec[name] + return getattr(spec, name) + + +def _subtracted_along_path( + graph: Mapping[str, VariableInfo], path: Sequence[str] +) -> tuple[str, ...]: + names: list[str] = [] + for ancestor in path[:-1]: + info = graph.get(ancestor) + if info is None: + continue + for edge, child in info.edges(): + if EDGE_SIGNS[edge] < 0: + names.append(child) + return tuple(names) + + +def _component( + node: Mapping[str, Any], + graph: Mapping[str, VariableInfo], + law_classifications: Mapping[str, Mapping[str, Any]], +) -> Component: + info = graph.get(node["name"]) + return Component( + name=node["name"], + label=node["label"], + documentation=info.documentation if info is not None else "", + entity=node["entity"], + sign=node["sign"], + path=tuple(node["path"]), + kind=node["kind"], + parameter_text=info.parameter_text if info is not None else "", + law_classification=law_classifications.get(node["name"]), + subtracted_along_path=_subtracted_along_path(graph, node["path"]), + ) + + +def _is_under(path: Sequence[str], prefixes: Iterable[tuple[str, ...]]) -> bool: + path = tuple(path) + return any(path[: len(prefix)] == prefix for prefix in prefixes) + + +def _nonzero(value: Any, tolerance: float) -> bool: + return isinstance(value, (int, float)) and abs(value) > tolerance + + +def _masking_ancestors( + graph: Mapping[str, VariableInfo], path: Sequence[str] +) -> tuple[str, ...]: + """Ancestors on ``path`` whose ``defined_for`` can zero what lies below them.""" + return tuple( + name + for name in path[:-1] + if graph.get(name) is not None and graph[name].defined_for is not None + ) + + +def _cells_for( + cells: Sequence[Mapping[str, Any]] | None, + variable: str, + scale: int, + tolerance: float, + masks: Sequence[str] = (), +) -> tuple[list[dict], bool | None, float]: + """Cells where ``variable`` reaches the reference, if that is known, and the sum. + + A value does not reach the reference in a cell where an ancestor in + ``masks`` (one with ``defined_for``) is zero. + """ + if cells is None: + return [], None, 0.0 + hits = [] + known = 0 + total = 0.0 + for cell in cells: + values = cell.get("values") or {} + if variable not in values or values[variable] is None: + continue + known += 1 + value = float(values[variable]) + if abs(value) <= tolerance: + continue + if any( + values.get(mask) is not None and abs(float(values[mask])) <= tolerance + for mask in masks + ): + continue + effect = scale * value + total += effect + reference = cell.get("reference") + hits.append( + { + "scenario_id": cell["scenario_id"], + "output_id": cell.get("output_id"), + "value": value, + "effect_on_reference": effect, + "reference": reference, + } + ) + if hits: + return hits, True, total + return hits, (False if known else None), total + + +def _conservation( + tree: Sequence[Mapping[str, Any]], + graph: Mapping[str, VariableInfo], + cells: Sequence[Mapping[str, Any]], + tolerance: float, +) -> dict[str, Any]: + expanded = [] + seen = set() + for node in tree: + if node["kind"] in ("root", "intermediate") and node["name"] not in seen: + seen.add(node["name"]) + expanded.append(node["name"]) + checked = 0 + failures = [] + masked = [] + for cell in cells: + values = cell.get("values") or {} + for name in expanded: + info = graph[name] + edges = info.edges() + if values.get(name) is None or any( + values.get(child) is None for _, child in edges + ): + continue + total = sum( + EDGE_SIGNS[edge] * float(values[child]) for edge, child in edges + ) + value = float(values[name]) + scale = abs(value) + sum(abs(float(values[child])) for _, child in edges) + checked += 1 + if abs(value - total) <= max(tolerance, 5e-7 * scale): + continue + record = { + "scenario_id": cell["scenario_id"], + "variable": name, + "value": value, + "sum_of_parts": total, + } + # The engine returns the default (0) wherever defined_for is not + # positive, whatever the parts sum to. + if info.defined_for is not None and abs(value) <= tolerance: + record["defined_for"] = info.defined_for + masked.append(record) + else: + failures.append(record) + return { + "expanded_nodes": expanded, + "checks": checked, + "failures": failures, + "masked_by_defined_for": masked, + } + + +def _household_mismatches( + output_id: str, + records: Sequence[Mapping[str, Any]], + tolerance: float, +) -> tuple[list[dict], list[dict]]: + mismatches = [] + notes = [] + for record in records: + deltas = record.get("output_deltas") or {} + own_return = record.get("own_return") or {} + own_outputs = own_return.get("outputs") or {} + if output_id not in deltas or output_id not in own_outputs: + continue + earned = float(record.get("earned_income") or 0.0) + unearned = float(record.get("unearned_income") or 0.0) + if earned + unearned <= tolerance: + continue + delta = float(deltas[output_id]) + if abs(delta) > tolerance: + continue + own = float(own_outputs[output_id]) + required = own_return.get("required_to_file") + counted_by = { + other: float(value) + for other, value in sorted(deltas.items()) + if other != output_id and abs(float(value)) > tolerance + } + person = record["person"] + entry = { + "output_id": output_id, + "rule": "household_scope", + "scenario_id": record["scenario_id"], + "person": person, + "earned_income": earned, + "unearned_income": unearned, + "reference": (record.get("references") or {}).get(output_id), + "change_without_person_income": delta, + "own_return_value": own, + "required_to_file": required, + "own_return_details": { + key: value for key, value in own_return.items() if key != "outputs" + }, + "counted_by": counted_by, + } + filing = { + True: "they would have a federal filing requirement", + False: "they would have no federal filing requirement", + }.get(required, "their filing requirement is unknown") + counters = ( + f"{', '.join(counted_by)} count{'s' if len(counted_by) == 1 else ''} it" + if counted_by + else "no other scored output of this household counts it" + ) + if abs(own) > tolerance: + entry["reason"] = ( + f"{person}'s income (earned {earned:,.2f}, unearned " + f"{unearned:,.2f}) is outside the reference's computation " + f"(removing it moves the reference by {delta:,.2f}); on their own " + f"return it would come to {own:,.2f}; {filing}; {counters}" + ) + mismatches.append(entry) + elif required: + entry["reason"] = ( + f"{person}'s income is outside the reference's computation and " + f"{filing}, but their own return comes to 0 for this output; " + f"{counters}" + ) + notes.append(entry) + return mismatches, notes + + +def _normalize_law( + law_classifications: Mapping[str, Mapping[str, Any]] | None, +) -> dict[str, Mapping[str, Any]]: + return dict(law_classifications or {}) + + +def conformance_report( + specs: Iterable[Any], + graph: Mapping[str, VariableInfo], + *, + cell_components: Mapping[str, Sequence[Mapping[str, Any]]] | None = None, + household_scope: Sequence[Mapping[str, Any]] | None = None, + household_prompt: bool = False, + law_classifications: Mapping[str, Mapping[str, Any]] | None = None, + rules: Sequence[QualifierRule] = QUALIFIER_RULES, + tolerance: float = TOLERANCE, +) -> dict[str, Any]: + """Test every output's components against the qualifiers its definition carries. + + ``specs`` are output specs (mappings or objects with ``id``, + ``pe_variable`` and ``prompt``, the definition). ``cell_components`` maps + an output id to its scored cells, each ``{"scenario_id", "output_id", + "reference", "values": {variable: value}}`` where a value is the + variable's total over the household's entities. ``household_scope`` holds + one record per household member whose income an engine run removed: + ``{"scenario_id", "person", "earned_income", "unearned_income", + "output_deltas": {output_id: reference minus value without the member's + income}, "references": {...}, "own_return": {"required_to_file", + "outputs": {output_id: value on the member's own return}}}``. + ``household_prompt`` says every output was asked for the household (the + frozen prompts' "Provide the following policy quantities for this + household"). ``law_classifications`` maps a variable to a cited + classification such as ``{"classification": + "optional_employer_pass_through", "citation": ...}``. + + The report depends only on the variables the outputs' sums reach and on + the arguments; it never reads the rest of the graph. + """ + law = _normalize_law(law_classifications) + outputs = [] + flat_mismatches: list[dict] = [] + flat_silent: list[dict] = [] + household_notes: list[dict] = [] + conservation_failures = 0 + for spec in specs: + output_id = str(_spec_field(spec, "id")) + root = str(_spec_field(spec, "pe_variable")) + definition = str(_spec_field(spec, "prompt")) + tree = component_tree(graph, root) + cells = None + if cell_components is not None: + cells = list(cell_components.get(output_id, ())) + detected = [ + rule + for rule in rules + if rule.detects(definition, household_prompt=household_prompt) + ] + components = [_component(node, graph, law) for node in tree] + rule_checks = [] + output_mismatches: list[dict] = [] + flagged: set[tuple[str, ...]] = set() + for rule in detected: + if rule.scope != "component" or rule.test is None: + continue + rule_flagged: set[tuple[str, ...]] = set() + hits = 0 + for node, component in zip(tree, components, strict=True): + if _is_under(node["path"], rule_flagged): + continue + hit = rule.test(component) + if hit is None: + continue + hits += 1 + path = tuple(node["path"]) + rule_flagged.add(path) + effect_variable = rule.effect(component) if rule.effect else None + masks = _masking_ancestors(graph, node["path"]) + if effect_variable is None: + effect_cells = _cells_for( + cells, node["name"], node["sign"], tolerance, masks + ) + else: + effect_cells = _cells_for( + cells, effect_variable, 1, tolerance, masks + ) + hit_cells, material, total = effect_cells + output_mismatches.append( + { + "output_id": output_id, + "rule": rule.id, + "component": node["name"], + "label": node["label"], + "path": node["path"], + "sign": node["sign"], + "entity": node["entity"], + "kind": node["kind"], + "evidence": hit.evidence, + "reason": hit.reason, + "effect_variable": effect_variable or node["name"], + "material": material, + "cells": hit_cells, + "total_effect_on_references": total, + } + ) + flagged |= rule_flagged + rule_checks.append( + { + "rule": rule.id, + "status": "mismatch" if hits else "conforms", + "components_tested": len(tree), + "mismatches": hits, + } + ) + + definition_tokens = _tokens(definition) + named: set[tuple[str, ...]] = set() + silent = [] + for node in tree: + path = tuple(node["path"]) + # A member of a parameter list is named by the list's category (the + # output is "all state refundable credits", say); any other node is + # named when its label is generic or shares a word with the + # definition. Everything under a named node counts as named. + if len(path) > 1 and ( + node["via_parameter"] or _is_named(node, definition_tokens) + ): + named.add(path) + if node["kind"] != "leaf" or len(path) == 1: + continue + if _is_under(path, flagged) or _is_under(path, named): + continue + if node["leaf_reason"] == "cycle": + continue + hit_cells, material, total = _cells_for( + cells, + node["name"], + node["sign"], + tolerance, + _masking_ancestors(graph, node["path"]), + ) + silent.append( + { + "output_id": output_id, + "component": node["name"], + "label": node["label"], + "path": node["path"], + "sign": node["sign"], + "material": material, + "cells": hit_cells, + "total_effect_on_references": total, + } + ) + + household_rule = next( + (rule for rule in detected if rule.scope == "household"), None + ) + household_mismatches: list[dict] = [] + if household_rule is not None and household_scope is not None: + household_mismatches, notes = _household_mismatches( + output_id, household_scope, tolerance + ) + household_notes.extend(notes) + rule_checks.append( + { + "rule": household_rule.id, + "status": "mismatch" if household_mismatches else "conforms", + "records_tested": len(household_scope), + "mismatches": len(household_mismatches), + } + ) + elif household_rule is not None: + rule_checks.append( + { + "rule": household_rule.id, + "status": "not_evaluated", + "records_tested": 0, + "mismatches": 0, + } + ) + + entry: dict[str, Any] = { + "id": output_id, + "pe_variable": root, + "definition": definition, + "qualifiers": [rule.id for rule in detected], + "tree": tree, + "rule_checks": rule_checks, + "mismatches": output_mismatches, + "household_scope_mismatches": household_mismatches, + "definition_silent": silent, + } + if cells is not None: + conservation = _conservation(tree, graph, cells, tolerance) + conservation_failures += len(conservation["failures"]) + entry["conservation"] = conservation + tree_names = {node["name"] for node in tree} + entry["cells"] = [ + { + "scenario_id": cell["scenario_id"], + "output_id": cell.get("output_id"), + "reference": cell.get("reference"), + "nonzero_components": { + name: float(value) + for name, value in sorted((cell.get("values") or {}).items()) + if name in tree_names and _nonzero(value, tolerance) + }, + } + for cell in cells + ] + outputs.append(entry) + flat_mismatches.extend(output_mismatches) + flat_mismatches.extend(household_mismatches) + flat_silent.extend(silent) + + by_rule: dict[str, dict[str, int]] = {} + material_cells: set[tuple[str, str]] = set() + for mismatch in flat_mismatches: + counts = by_rule.setdefault( + mismatch["rule"], {"mismatches": 0, "material": 0, "cells": 0} + ) + counts["mismatches"] += 1 + if mismatch["rule"] == "household_scope": + counts["material"] += 1 + counts["cells"] += 1 + material_cells.add((mismatch["scenario_id"], mismatch["output_id"])) + continue + if mismatch["material"]: + counts["material"] += 1 + counts["cells"] += len(mismatch["cells"]) + for cell in mismatch["cells"]: + material_cells.add( + (cell["scenario_id"], cell.get("output_id") or mismatch["output_id"]) + ) + return { + "schema_version": SCHEMA_VERSION, + "household_prompt": household_prompt, + "tolerance": tolerance, + "rules": [ + { + "id": rule.id, + "description": rule.description, + "pattern": rule.pattern, + "scope": rule.scope, + } + for rule in rules + ], + "outputs": outputs, + "mismatches": flat_mismatches, + "definition_silent": [item for item in flat_silent if item["material"]], + "household_scope_notes": household_notes, + "household_scope_records": list(household_scope or ()), + "summary": { + "outputs": len(outputs), + "mismatches": len(flat_mismatches), + "material_mismatches": sum( + 1 + for mismatch in flat_mismatches + if mismatch.get("material") or mismatch["rule"] == "household_scope" + ), + "material_cells": len(material_cells), + "by_rule": by_rule, + "definition_silent_material": sum( + 1 for item in flat_silent if item["material"] + ), + "conservation_failures": conservation_failures, + }, + } + + +# ----- Markdown ----- # + + +def _money(value: Any) -> str: + if value is None: + return "" + return f"{float(value):,.2f}" + + +def _cell_list(cells: Sequence[Mapping[str, Any]], limit: int = 6) -> str: + parts = [ + f"{cell['scenario_id']} {_money(cell['effect_on_reference'])}" + for cell in cells[:limit] + ] + if len(cells) > limit: + parts.append(f"... {len(cells) - limit} more") + return "; ".join(parts) + + +def _escape(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def render_markdown( + report: Mapping[str, Any], *, title: str, notes: Sequence[str] = () +) -> str: + """A short Markdown summary of a report's mismatches, then ``notes``.""" + summary = report["summary"] + lines = [ + f"# {title}", + "", + f"{summary['outputs']} outputs; {summary['mismatches']} mismatches, " + f"{summary['material_mismatches']} material, touching " + f"{summary['material_cells']} scored cells; " + f"{summary['definition_silent_material']} material definition-silent " + f"components; {summary['conservation_failures']} conservation failures.", + "", + "## Component mismatches", + "", + "| Output | Rule | Component | Evidence | Material | Cells (effect on " + "reference) | Reason |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + component_rows = [m for m in report["mismatches"] if m["rule"] != "household_scope"] + component_rows.sort( + key=lambda m: (not m["material"], m["output_id"], m["rule"], m["component"]) + ) + for mismatch in component_rows: + material = {True: "yes", False: "no", None: "unknown"}[mismatch["material"]] + lines.append( + f"| {mismatch['output_id']} | {mismatch['rule']} | " + f"`{mismatch['component']}` | {mismatch['evidence']} | {material} | " + f"{_cell_list(mismatch['cells'])} | {_escape(mismatch['reason'])} |" + ) + if not component_rows: + lines.append("| none | | | | | | |") + household_rows = [m for m in report["mismatches"] if m["rule"] == "household_scope"] + lines += [ + "", + "## Household scope", + "", + "| Scenario | Output | Person | Earned | Unearned | Reference | Own return | " + "Must file | Counted by |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for mismatch in sorted( + household_rows, key=lambda m: (m["scenario_id"], m["output_id"]) + ): + counted = ", ".join( + f"{name} {_money(value)}" for name, value in mismatch["counted_by"].items() + ) + lines.append( + f"| {mismatch['scenario_id']} | {mismatch['output_id']} | " + f"{mismatch['person']} | {_money(mismatch['earned_income'])} | " + f"{_money(mismatch['unearned_income'])} | " + f"{_money(mismatch['reference'])} | " + f"{_money(mismatch['own_return_value'])} | " + f"{mismatch['required_to_file']} | {counted} |" + ) + if not household_rows: + lines.append("| none | | | | | | | | |") + lines += [ + "", + "## Material definition-silent components (information)", + "", + "| Output | Component | Label | Cells (effect on reference) |", + "| --- | --- | --- | --- |", + ] + for item in report["definition_silent"]: + lines.append( + f"| {item['output_id']} | `{item['component']}` | " + f"{_escape(item['label'])} | {_cell_list(item['cells'])} |" + ) + if not report["definition_silent"]: + lines.append("| none | | | |") + if notes: + lines += ["", "## Notes", ""] + lines += [f"- {note}" for note in notes] + lines.append("") + return "\n".join(lines) diff --git a/policybench/publication_sources.py b/policybench/publication_sources.py new file mode 100644 index 00000000..b59ff8aa --- /dev/null +++ b/policybench/publication_sources.py @@ -0,0 +1,1906 @@ +"""Check whether each 2026 parameter value behind a scored reference was published. + +A PolicyBench reference follows from the stated facts and from law published +before the 2026-07-03 reference freeze. A reference can still rest on a number +nobody published before the freeze: an amount PolicyEngine computed itself +(uprated from an index when the tree loads, or typed into the YAML as a 2026 +entry after doing the arithmetic by hand) and cited to a statute, an index +release or guidance for another year. Louisiana's 2026 standard deduction is +the case that prompted this check: policyengine-us 2.15.17 encodes $12,835 as +an explicit 2026 entry whose YAML comment shows the CPI-U arithmetic +(``12,500 * (324.054 / 315.605)``), and its four citations are a statute, a +withholding-tables notice, a CPI release and the 2025 return instructions, none +of which this module counts as a 2026 publication of the amount. + +The engine-side script +(``reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py``) +records which parameter values each scored reference reads and where each 2026 +value comes from. This module holds the pure part: classifying citations, +reading dates out of citation titles and links, deciding a value's origin and +its flags, and rendering the report. Nothing here imports policyengine. + +Origins (``classify_origin``), decided against the plain policyengine-us system: + +``explicit_2026`` + A YAML entry dated in 2026 holds the value. +``carried_forward`` + A YAML entry dated before 2026 holds the value (static law). +``uprated`` / ``interpolated`` / ``backdated`` / ``computed_at_load`` + The value was produced while the parameter tree loaded: by core's + ``uprate_parameters``, by interpolation, by policyengine-us's + ``backdate_parameters`` (which copies a parameter's earliest value, here one + dated after the instant read, back to 2015), or by another + ``Parameter.update`` that policyengine-us code runs at load (for example its + uprating extensions). +``created_at_load`` + The parameter has no YAML source at all (a structural default). +``convention`` + The reference conventions (``latest_final``) set the value. +``unresolved`` + None of the above explains the value. + +Citations (``classify_citation``) are classified by source (government, +law republisher, other) and kind (statute, regulation, register notice, +agency publication, withholding guidance, index data, other), and dated where +the title or link states a date. A statute or regulation citation dates the law, +not the 2026 amount: it can say an amount is indexed without stating what the +indexed amount is, and a date it states is usually an effective date, so it has +no freeze status. Only a government register notice or agency publication +issued for 2026 (it applies to 2026, or was issued in 2025 or 2026 and names no +other year) and not dated after the freeze counts as a 2026 publication; for an +economic series (a price index or projection) a government index release +counts too. Whether that publication actually states the amount is not +something metadata can show; this check only finds values that have no such +candidate at all. + +Flags (``value_flags``): + +``computed_or_indexed`` + The value was produced at load (uprated, interpolated, backdated, + computed). +``explicit_2026_without_2026_publication`` + An explicit 2026 entry with no 2026 publication among its citations. +``computed_in_yaml_comment`` + A YAML comment beside the entry shows PolicyEngine computed the value (an + arithmetic expression that reproduces it, projection or estimate wording, + or an index named beside a multiplication). +``created_at_load`` / ``origin_unresolved`` + The value has no YAML entry, or its origin could not be determined. +``no_government_citation`` + No citation is a government source. +``post_freeze_citation`` + At least one citation is dated after the freeze and none before it. +``convention_set`` + ``latest_final`` supplied the value; no other flag is set. +``carried_forward`` + Informational. +""" + +from __future__ import annotations + +import ast +import json +import operator +import re +from collections import Counter +from dataclasses import asdict, dataclass +from datetime import date +from typing import Any, Iterable, Mapping, Sequence +from urllib.parse import unquote, urlsplit + +FREEZE_DATE = date(2026, 7, 3) +REFERENCE_YEAR = 2026 + +ORIGINS = ( + "explicit_2026", + "carried_forward", + "uprated", + "interpolated", + "backdated", + "computed_at_load", + "created_at_load", + "unresolved", + "convention", +) +COMPUTED_ORIGINS = frozenset( + {"uprated", "interpolated", "backdated", "computed_at_load"} +) +# policyengine-us's tools/parameters.py function that copies each parameter's +# earliest value back to 2015 when the tree loads. +BACKDATE_WRITER = "backdate_parameters" + +FLAGS = ( + "computed_or_indexed", + "explicit_2026_without_2026_publication", + "computed_in_yaml_comment", + "created_at_load", + "origin_unresolved", + "no_government_citation", + "post_freeze_citation", + "convention_set", + "carried_forward", +) +INFORMATIONAL_FLAGS = frozenset({"convention_set", "carried_forward"}) +REVIEW_FLAGS = tuple(flag for flag in FLAGS if flag not in INFORMATIONAL_FLAGS) + +SOURCE_CLASSES = ("government", "law_republisher", "other") +CITATION_KINDS = ( + "index_data", + "withholding", + "regulation", + "register", + "agency_publication", + "statute", + "other", +) +QUALIFYING_KINDS = frozenset({"register", "agency_publication"}) +CATEGORIES = ("law", "economic_series", "model_switch") + +# Hosts outside .gov, .us and .mil that the citations of the engine run point +# to and that are the official site of a US government body (named here from +# the site itself). A host left out only makes a citation count as "other". +GOVERNMENT_HOSTS = frozenset( + { + "ctoec.org", # Connecticut Office of Early Childhood + "ctpaidleave.org", # Connecticut Paid Leave Authority + "fldoe.org", # Florida Department of Education + "flrules.org", # Florida Administrative Code (Department of State) + "kslegislature.org", # Kansas Legislature + "louisianabelieves.com", # Louisiana Department of Education + "marylandpublicschools.org", # Maryland State Department of Education + "myflfamilies.com", # Florida Department of Children and Families + "myflorida.com", # Florida agencies (for example ahca.myflorida.com) + "okdhslive.org", # Oklahoma Human Services policy library + "oscn.net", # Oklahoma State Courts Network (statutes) + "sccgov.org", # County of Santa Clara + "sfhsa.org", # San Francisco Human Services Agency + } +) +# Commercial or nonprofit republishers of statutes and regulations that the +# citations point to. +LAW_REPUBLISHER_HOSTS = frozenset( + { + "amlegal.com", + "casetext.com", + "findlaw.com", + "justia.com", + "law.cornell.edu", + "legiscan.com", + "lexis.com", + "public.law", + "trackbill.com", + "westlaw.com", + } +) +INDEX_HOSTS = frozenset({"bls.gov", "cbo.gov"}) +GOVERNMENT_SUFFIXES = (".gov", ".mil", ".us") + +MODEL_SWITCH_PREFIXES = ("gov.abolitions.", "gov.contrib.", "gov.simulation.") +ECONOMIC_SERIES_PREFIXES = ("gov.bls.", "gov.cbo.", "calibration.", "gov.census.") + +_MONTHS = { + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, +} +_MONTH = ( + r"(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|June?|July?|" + r"Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)" +) +_YEAR = r"((?:19|20)\d{2})" +_FULL_DATE = re.compile( + rf"\b{_MONTH}\.?\s+(\d{{1,2}})(?:st|nd|rd|th)?,?\s+{_YEAR}(?!\d)", re.IGNORECASE +) +_DAY_FIRST_DATE = re.compile( + rf"\b(\d{{1,2}})\s+{_MONTH}\.?,?\s+{_YEAR}(?!\d)", re.IGNORECASE +) +_ISO_DATE = re.compile(rf"(? dict[str, str]: + data = asdict(self) + data["earliest"] = self.earliest.isoformat() + data["latest"] = self.latest.isoformat() + return data + + +_PRECISION_RANK = {"day": 3, "month": 2, "year": 1} + + +def _month_number(name: str) -> int: + return _MONTHS[name[:3].lower()] + + +def _safe_date(year: int, month: int, day: int) -> date | None: + try: + return date(year, month, day) + except ValueError: + return None + + +def _month_end(year: int, month: int) -> date: + if month == 12: + return date(year, 12, 31) + return date.fromordinal(date(year, month + 1, 1).toordinal() - 1) + + +def _year_evidence(year: int, basis: str) -> DateEvidence: + return DateEvidence(date(year, 1, 1), date(year, 12, 31), "year", basis) + + +def _fr_year(volume: int) -> int | None: + """The calendar year of a Federal Register volume (volume 1 is 1936).""" + year = 1935 + volume + return year if 1936 <= year <= 2099 else None + + +def stated_years(text: str | None) -> set[int]: + """Every year the text states. + + That is each four-digit 19xx or 20xx run (also inside longer digit runs, + such as the 2026 of ``cpi_01132026``), the year of each Federal Register + volume cited as `` FR ``, and the year of each two-digit + issuance number the date reader understands (IRS ``rp-25-32`` file names + and Louisiana ``RIB 26-019`` bulletins). + """ + if not text: + return set() + years = {int(match.group(1)) for match in _ANY_YEAR.finditer(text)} + for match in _FR_CITATION.finditer(text): + year = _fr_year(int(match.group(1))) + if year is not None: + years.add(year) + for pattern in (_IRS_DROP, _RIB): + years |= {2000 + int(match.group(1)) for match in pattern.finditer(text)} + return years + + +def _date_candidates(text: str) -> list[DateEvidence]: + found: list[DateEvidence] = [] + for match in _FULL_DATE.finditer(text): + day = _safe_date( + int(match.group(3)), _month_number(match.group(1)), int(match.group(2)) + ) + if day: + found.append(DateEvidence(day, day, "day", match.group(0))) + for match in _DAY_FIRST_DATE.finditer(text): + day = _safe_date( + int(match.group(3)), _month_number(match.group(2)), int(match.group(1)) + ) + if day: + found.append(DateEvidence(day, day, "day", match.group(0))) + for match in _ISO_DATE.finditer(text): + day = _safe_date(int(match.group(1)), int(match.group(3)), int(match.group(4))) + if day: + found.append(DateEvidence(day, day, "day", match.group(0))) + for match in _EIGHT_DIGITS.finditer(text): + digits = match.group(1) + day = None + year = int(digits[:4]) + if 1990 <= year <= 2035: + day = _safe_date(year, int(digits[4:6]), int(digits[6:])) + if day is None: + year = int(digits[4:]) + if 1990 <= year <= 2035: + day = _safe_date(year, int(digits[:2]), int(digits[2:4])) + if day: + found.append(DateEvidence(day, day, "day", digits)) + for match in _MONTH_YEAR.finditer(text): + year, month = int(match.group(2)), _month_number(match.group(1)) + found.append( + DateEvidence( + date(year, month, 1), _month_end(year, month), "month", match.group(0) + ) + ) + for match in _PATH_YEAR_MONTH.finditer(text): + year, month = int(match.group(1)), int(match.group(2)) + if 1 <= month <= 12: + found.append( + DateEvidence( + date(year, month, 1), + _month_end(year, month), + "month", + match.group(0), + ) + ) + for match in _FR_CITATION.finditer(text): + year = _fr_year(int(match.group(1))) + if year is not None: + found.append(_year_evidence(year, match.group(0))) + for pattern in (_IRS_ISSUANCE, _IRB_PATH): + for match in pattern.finditer(text): + found.append(_year_evidence(int(match.group(1)), match.group(0))) + for pattern in (_IRS_DROP, _RIB): + for match in pattern.finditer(text): + found.append(_year_evidence(2000 + int(match.group(1)), match.group(0))) + return found + + +def extract_publication_date(text: str | None) -> DateEvidence | None: + """The issue date a citation's title or link states, if it states one. + + Full dates (``February 20, 2026``, ``2026-02-20``, ``/2025/10/09/``, an + eight-digit ``20260113`` or ``01132026`` run), month and year + (``October 2025``), Federal Register volumes and IRS issuance numbers + (``Rev. Proc. 2025-32``, ``rp-25-32``, ``IRB 2025-45``) and Louisiana + revenue bulletins (``RIB 26-019``) are read. The most precise evidence + wins, and among equally precise evidence the latest. A stated effective + date can be read as an issue date. The result never lies after the last + year the text states (``stated_years``). + """ + if not text: + return None + candidates = _date_candidates(text) + if not candidates: + return None + return max( + candidates, + key=lambda item: (_PRECISION_RANK[item.precision], item.latest, item.basis), + ) + + +def extract_applies_to_year(title: str | None, href: str | None) -> int | None: + """The year a document applies to (a tax, fiscal or benefit year), if stated. + + ``Tax Year 2026``, ``FY 2026``, ``2025 Form 540 Instructions``, + ``Publication 15-T (2026)``, ``for 2026`` and form links such as + ``/forms/2025/`` or ``WEB(2025)`` are read; a title that states exactly one + year and no issue date is read as applying to that year. When several + years are stated, the latest wins. + """ + title = title or "" + href = unquote(href or "") + years: list[int] = [] + for pattern in (_APPLIES_LABELLED, _APPLIES_YEAR_FIRST, _APPLIES_FOR): + for match in pattern.finditer(title): + years += [int(group) for group in match.groups() if group] + years += [int(match.group(1)) for match in _APPLIES_PAREN.finditer(title)] + if not years: + for pattern in _APPLIES_HREF: + years += [int(match.group(1)) for match in pattern.finditer(href)] + if not years and extract_publication_date(title) is None: + bare = {int(match.group(1)) for match in _FOUR_DIGIT_YEAR.finditer(title)} + if len(bare) == 1: + years = list(bare) + return max(years) if years else None + + +def pre_freeze_status( + evidence: DateEvidence | None, + applies_to_year: int | None, + freeze: date = FREEZE_DATE, +) -> bool | None: + """True if a citation was certainly issued before the freeze, False if + certainly on or after it, None if that is unknown. + + A stated issue date decides. Without one, a document that applies to year + Y is taken to be issued between January 1 of Y-1 and December 31 of Y+1 + (a 2026 form can appear in late 2025; instructions for 2026 appear in + 2027). + """ + if evidence is not None: + earliest, latest = evidence.earliest, evidence.latest + elif applies_to_year is not None: + earliest = date(applies_to_year - 1, 1, 1) + latest = date(applies_to_year + 1, 12, 31) + else: + return None + if latest < freeze: + return True + if earliest >= freeze: + return False + return None + + +# --------------------------------------------------------------------------- +# Citations +# --------------------------------------------------------------------------- + + +def normalize_references(raw: Any, level: str = "parameter") -> list[dict]: + """A parameter's ``metadata.reference`` as a list of ``{title, href, level}``. + + Accepts a list or a single item; each item may be a ``{title, href}`` + mapping or a bare string (a link if it looks like one, else a title). + """ + if raw is None: + return [] + items = raw if isinstance(raw, (list, tuple)) else [raw] + found = [] + for item in items: + if isinstance(item, Mapping): + title = item.get("title") + href = item.get("href") + elif isinstance(item, str): + looks_like_link = item.strip().lower().startswith(("http", "www.")) + title, href = (None, item) if looks_like_link else (item, None) + else: + continue + title = str(title).strip() if title not in (None, "") else None + href = str(href).strip() if href not in (None, "") else None + if title is None and href is None: + continue + found.append({"title": title, "href": href, "level": level}) + return found + + +def citation_host(href: str | None) -> str | None: + """The lower-case host of a citation link (the archived link for Wayback).""" + if not href: + return None + text = href.strip() + if not re.match(r"^[a-z][a-z0-9+.-]*://", text, re.IGNORECASE): + text = "http://" + text + try: + host = (urlsplit(text).hostname or "").lower() + except ValueError: + return None + if host == "web.archive.org": + inner = re.search(r"/web/[^/]*/(.+)$", text) + if inner: + return citation_host(inner.group(1)) + return host[4:] if host.startswith("www.") else host or None + + +def _host_matches(host: str, domains: Iterable[str]) -> bool: + return any(host == domain or host.endswith("." + domain) for domain in domains) + + +def source_class(title: str | None, href: str | None) -> str: + """``government``, ``law_republisher`` or ``other``, from the link's host. + + Without a link, a title naming federal law, the Federal Register or an IRS + issuance counts as a government source. + """ + host = citation_host(href) + if host: + if _host_matches(host, LAW_REPUBLISHER_HOSTS): + return "law_republisher" + if host.endswith(GOVERNMENT_SUFFIXES) or _host_matches(host, GOVERNMENT_HOSTS): + return "government" + return "other" + title = title or "" + if re.search( + r"\bU\.?\s?S\.?\s?C\b|\bC\.?\s?F\.?\s?R\b|federal\s+register|public\s+law|" + r"\b\d{1,3}\s+(?:FR|Fed\.?\s*Reg\.?)\s+\d|" + r"internal\s+revenue|\bIRS\b|rev(?:enue)?\.?\s*proc", + title, + re.IGNORECASE, + ): + return "government" + return "other" + + +def citation_kind(title: str | None, href: str | None) -> str: + """What a citation is: index data, withholding guidance, regulation, + register notice, agency publication, statute, or other. + + Checked in that order of precedence, except that strong agency-publication + words (``Rev. Proc.``, ``Form``, ``Instructions``, ``Bulletin``...) win over + statute words and weak ones (``rates``, ``standards``...) lose to them. + """ + title = title or "" + link = unquote(href or "") + host = citation_host(href) or "" + text = f"{title} {link}" + if _host_matches(host, INDEX_HOSTS) or _INDEX_TITLE.search(title): + if not _AGENCY_STRONG.search(title) or _host_matches(host, INDEX_HOSTS): + return "index_data" + if _WITHHOLDING.search(text): + return "withholding" + if _host_matches(host, _REGULATION_HOSTS) or _REGULATION.search(title): + return "regulation" + if _host_matches(host, _REGISTER_HOSTS) or _REGISTER.search(title): + return "register" + if _AGENCY_STRONG.search(title): + return "agency_publication" + if _STATUTE_HOST.search(host) or _STATUTE.search(title): + return "statute" + if _AGENCY_WEAK.search(title) or any(part in host for part in _AGENCY_HOST_PARTS): + return "agency_publication" + if source_class(title, href) == "government": + return "agency_publication" + return "other" + + +def classify_citation( + title: str | None, href: str | None = None, level: str = "parameter" +) -> dict[str, Any]: + """Classify one citation: source, kind, stated dates and freeze status. + + ``kind`` is ``citation_kind``, except that a register notice or agency + publication on a host that is not a government source counts as + ``other``. ``pre_freeze`` is ``pre_freeze_status`` of the stated dates, + except for a statute or regulation, whose stated date is usually an + effective date or a code edition: its status is None. + ``dated_for_2026`` is true when the citation applies to 2026, or was issued + in 2025 or 2026 and names no other year it applies to. + ``dated_2026_publication`` is true for a government register notice or + agency publication dated for 2026 and not dated after the freeze. + ``recent_law`` is true for a statute or regulation that states 2025 or 2026 + (a session, a bill year) or names the 119th Congress. + """ + title = None if title in (None, "") else str(title) + href = None if href in (None, "") else str(href) + text = " ".join(part for part in (title, unquote(href or "")) if part) + evidence = extract_publication_date(text) + applies = extract_applies_to_year(title, href) + source = source_class(title, href) + kind = citation_kind(title, href) + if source != "government" and kind in QUALIFYING_KINDS: + kind = "other" # a news story or advocacy page is not an agency's own + # A date a statute or regulation citation states is usually an effective + # date or a code edition, not when the law was published. + if kind in ("statute", "regulation"): + pre_freeze = None + else: + pre_freeze = pre_freeze_status(evidence, applies) + issued_year = ( + evidence.latest.year + if evidence is not None and evidence.earliest.year == evidence.latest.year + else None + ) + if applies is not None: + for_2026 = applies == REFERENCE_YEAR + else: + for_2026 = issued_year in (REFERENCE_YEAR - 1, REFERENCE_YEAR) + recent = (REFERENCE_YEAR - 1, REFERENCE_YEAR) + law_years = stated_years(text) | congress_years(text) + return { + "title": title, + "href": href, + "level": level, + "host": citation_host(href), + "source_class": source, + "kind": kind, + "publication_date": None if evidence is None else evidence.to_dict(), + "applies_to_year": applies, + "pre_freeze": pre_freeze, + "dated_for_2026": bool(for_2026), + "dated_2026_publication": bool( + source == "government" + and kind in QUALIFYING_KINDS + and pre_freeze is not False + and for_2026 + ), + "recent_law": bool( + kind in ("statute", "regulation") and law_years & set(recent) + ), + } + + +def congress_years(text: str | None) -> set[int]: + """The two calendar years of each Congress a citation names. + + ``119th-congress`` or ``119th Congress`` (as in a congress.gov link or + title), and public law numbers (``Pub. L. No. 119-21``, ``P.L. 119-21``, + ``PLAW-119publ21``), name the 119th Congress, which sits in 2025 and 2026. + """ + if not text: + return set() + years = set() + for match in _CONGRESS.finditer(text): + number = int(next(group for group in match.groups() if group)) + if 1 <= number <= 200: + years |= {1787 + 2 * number, 1788 + 2 * number} + return years + + +SUPPORT_TIERS = ( + "2026_publication", + "undated_publication", + "recent_law", + "inputs_only", + "other", +) + + +def supports_2026_value(citation: Mapping[str, Any], category: str = "law") -> bool: + """Whether a classified citation is a candidate publication of a 2026 value. + + For an economic series (a price index or projection) the index release + itself publishes the value, so a government index-data citation dated for + 2026 counts too. + """ + if citation.get("dated_2026_publication"): + return True + return bool( + category == "economic_series" + and citation.get("kind") == "index_data" + and citation.get("source_class") == "government" + and citation.get("pre_freeze") is not False + and citation.get("dated_for_2026") + ) + + +def explicit_support( + citations: Sequence[Mapping[str, Any]], category: str = "law" +) -> str: + """The strongest support an explicit 2026 entry's citations give its value. + + ``2026_publication``: a candidate 2026 publication (``supports_2026_value``). + ``undated_publication``: a government register notice or agency + publication that states no date or year (often a page listing every + year's amounts). ``recent_law``: a statute or regulation from 2025 or 2026, + which may state the amount itself. ``inputs_only``: withholding guidance or + index data for 2025 or 2026, which are inputs to an amount, not the amount. + ``other``: none of these. + """ + if any(supports_2026_value(c, category) for c in citations): + return "2026_publication" + if any( + c.get("source_class") == "government" + and c.get("kind") in QUALIFYING_KINDS + and c.get("publication_date") is None + and c.get("applies_to_year") is None + for c in citations + ): + return "undated_publication" + if any(c.get("recent_law") for c in citations): + return "recent_law" + if any( + c.get("kind") in ("withholding", "index_data") and c.get("dated_for_2026") + for c in citations + ): + return "inputs_only" + return "other" + + +# --------------------------------------------------------------------------- +# YAML entries and comments +# --------------------------------------------------------------------------- + + +def parameter_key_path(name: str, file_root: str) -> list[str] | None: + """Where a parameter sits inside the YAML file that defines ``file_root``. + + ``gov.x.amount.SINGLE`` in ``gov.x.amount`` is ``["SINGLE"]``; a scale + leaf ``gov.x.rates[1].threshold`` in ``gov.x.rates`` is + ``["[1]", "threshold"]``; the file's own parameter is ``[]``. Returns + None when the name is not inside the file. + """ + if name == file_root: + return [] + if not file_root: + rest = name + elif name.startswith(file_root + ".") or name.startswith(file_root + "["): + rest = name[len(file_root) :] + else: + return None + return _NAME_TOKEN.findall(rest) + + +def ancestor_names(name: str) -> list[str]: + """Every proper ancestor of a parameter name, nearest first. + + ``gov.x.rates[1].threshold`` gives ``gov.x.rates[1]``, ``gov.x.rates``, + ``gov.x`` and ``gov``. + """ + names = [] + for match in re.finditer(r"\.|\[", name): + names.append(name[: match.start()]) + return [ancestor for ancestor in reversed(names) if ancestor] + + +def _inline_comment(line: str) -> str | None: + quote = None + for index, char in enumerate(line): + if quote: + if char == quote: + quote = None + elif char in "'\"": + quote = char + elif char == "#" and (index == 0 or line[index - 1] in " \t"): + return line[index + 1 :].strip() + return None + + +def entry_comments(yaml_text: str, key_path: Sequence[str], instant: str) -> list[str]: + """The comments written beside one dated entry of a parameter's YAML. + + The entry is the line whose key is ``instant`` under ``key_path`` (as + ``parameter_key_path`` gives it; ``values`` and ``brackets`` wrappers are + skipped). Its comments are the comment lines directly above it and any + comment at the end of the line. Returns an empty list if the entry is not + found. + """ + target = [*key_path, instant] + lines = yaml_text.splitlines() + stack: list[tuple[int, str]] = [] + counters: dict[tuple[str, ...], int] = {} + for number, line in enumerate(lines): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(line) - len(line.lstrip(" ")) + content = stripped + dash = content.startswith("- ") or content == "-" + while stack and ( + stack[-1][0] > indent + or (stack[-1][0] == indent and (not dash or stack[-1][1].startswith("["))) + ): + stack.pop() + if dash: + parent = tuple(token for _, token in stack) + index = counters.get(parent, 0) + counters[parent] = index + 1 + stack.append((indent, f"[{index}]")) + content = content[1:].lstrip() + indent += 2 + if not content: + continue + match = _KEY_LINE.match(content) + if not match: + continue + key = match.group(1).strip().strip("'\"") + stack.append((indent, key)) + path = [token for _, token in stack if token not in ("values", "brackets")] + if path != target: + continue + comments = [] + above = number - 1 + while above >= 0 and lines[above].strip().startswith("#"): + comments.append(lines[above].strip().lstrip("#").strip()) + above -= 1 + comments.reverse() + inline = _inline_comment(content) + if inline: + comments.append(inline) + return [comment for comment in comments if comment] + return [] + + +def _evaluate(node: ast.AST) -> float: + if isinstance(node, ast.Expression): + return _evaluate(node.body) + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + if isinstance(node.value, bool): + raise ValueError("bool") + return float(node.value) + if isinstance(node, ast.BinOp) and type(node.op) in _BINARY_OPERATORS: + return _BINARY_OPERATORS[type(node.op)]( + _evaluate(node.left), _evaluate(node.right) + ) + if isinstance(node, ast.UnaryOp) and type(node.op) in _UNARY_OPERATORS: + return _UNARY_OPERATORS[type(node.op)](_evaluate(node.operand)) + raise ValueError(f"unsupported expression: {ast.dump(node)}") + + +def evaluate_arithmetic(text: str) -> list[tuple[str, float]]: + """Arithmetic expressions written in a comment, with their values. + + Only numbers (with ``,`` or ``_`` thousands separators), ``+ - * / × ÷`` + and parentheses are read; an expression needs a multiplication or a + division. Anything else is ignored, never executed. + """ + found = [] + for match in _EXPRESSION.finditer(text): + raw = match.group(0).strip() + if not re.search(r"[*/×÷]", raw) or len(re.findall(r"\d+", raw)) < 2: + continue + expression = _THOUSANDS.sub("", raw).replace("×", "*").replace("÷", "/") + while expression.count("(") > expression.count(")") and "(" in expression: + expression = expression[expression.index("(") + 1 :] + while expression.count(")") > expression.count("(") and ")" in expression: + expression = expression[: expression.rindex(")")] + try: + value = _evaluate(ast.parse(expression.strip(), mode="eval")) + except (SyntaxError, ValueError, ZeroDivisionError, RecursionError): + continue + found.append((raw, value)) + return found + + +def _numeric(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def comment_computation(comments: Sequence[str], value: Any) -> dict | None: + """Whether an entry's comments show PolicyEngine computed its value. + + ``computation`` is true when an arithmetic expression in a comment rounds + to the value (within 0.51; only for values of at least 1, since a + statutory fraction such as ``2/37`` is not a computed amount), when a + comment calls the value a projection or an estimate, or when it names an + index (CPI, uprating) beside a multiplication (``times``, ``*``). An index + named alone can describe how an agency set its published amount, so it + does not count. Returns None when there are no comments. + """ + if not comments: + return None + text = " ".join(comments) + projection = sorted( + {match.group(0).lower() for match in _PROJECTION_WORDS.finditer(text)} + ) + index = sorted({match.group(0).lower() for match in _INDEX_WORDS.finditer(text)}) + number = _numeric(value) + expressions = [] + for raw, result in evaluate_arithmetic(text): + matches = ( + number is not None and abs(number) >= 1 and abs(result - number) <= 0.51 + ) + expressions.append({"text": raw, "result": result, "matches_value": matches}) + multiplied = bool(_MULTIPLICATION_WORDS.search(text)) + return { + "comments": list(comments), + "keywords": sorted(set(projection) | set(index)), + "expressions": expressions, + "computation": bool(projection) + or (bool(index) and multiplied) + or any(e["matches_value"] for e in expressions), + } + + +# --------------------------------------------------------------------------- +# Origins and flags +# --------------------------------------------------------------------------- + + +def same_value(a: Any, b: Any) -> bool: + """Parameter values compared as the engine stores them (floats to 1e-9).""" + if isinstance(a, bool) or isinstance(b, bool): + return a == b + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + return a == b or abs(a - b) <= 1e-9 * max(1.0, abs(a), abs(b)) + if isinstance(a, list) and isinstance(b, list): + return len(a) == len(b) and all(same_value(x, y) for x, y in zip(a, b)) + return a == b + + +def classify_origin( + *, + entry_instant: str | None, + value: Any, + yaml_entries: Mapping[str, Mapping[str, Any]] | None, + uprated_instants: Iterable[str], + interpolated: bool, + load_updates: Sequence[Mapping[str, Any]], + reference_value: Any, + convention_modules: Sequence[str], + year: int = REFERENCE_YEAR, +) -> tuple[str, str, dict]: + """A value's origin, its origin in the plain system, and the evidence. + + ``entry_instant`` and ``value`` describe the plain system's entry in effect; + ``yaml_entries`` maps each date the parameter's own YAML lists to + ``{"value": ...}`` (None when the parameter has no YAML source); + ``uprated_instants`` are the dates core's uprating added; + ``load_updates`` are ``Parameter.update`` calls run while the plain tree + loaded that cover the instant. The value is convention-set when a + ``latest_final`` part's update covers the instant or the reference value + differs from the plain one. + """ + detail: dict[str, Any] = {} + uprated = set(uprated_instants) + if entry_instant is None: + baseline = "unresolved" + detail["reason"] = "no entry is in effect" + elif ( + yaml_entries is not None + and entry_instant in yaml_entries + and same_value(yaml_entries[entry_instant].get("value"), value) + ): + detail["yaml_entry"] = entry_instant + if entry_instant.startswith(str(year)): + baseline = "explicit_2026" + elif entry_instant < f"{year}-01-01": + baseline = "carried_forward" + else: + baseline = "unresolved" + detail["reason"] = "the entry in effect is dated after the year" + elif entry_instant in uprated: + baseline = "uprated" + detail["uprated_entry"] = entry_instant + elif interpolated: + baseline = "interpolated" + elif load_updates: + writers = sorted({str(item.get("caller")) for item in load_updates}) + detail["writers"] = writers + if any(BACKDATE_WRITER in writer for writer in writers): + baseline = "backdated" + if yaml_entries: + detail["backdated_from"] = min(yaml_entries) + else: + baseline = "computed_at_load" + elif yaml_entries is None: + baseline = "created_at_load" + else: + baseline = "unresolved" + detail["reason"] = f"the YAML lists no entry at {entry_instant} with this value" + convention = bool(convention_modules) or not same_value(reference_value, value) + if convention: + detail["convention_modules"] = list(convention_modules) + return ("convention" if convention else baseline), baseline, detail + + +def value_flags( + origin: str, + citations: Sequence[Mapping[str, Any]], + *, + convention_modules: Sequence[str] = (), + comment_evidence: Mapping[str, Any] | None = None, + category: str = "law", +) -> list[str]: + """The flags of one (parameter, 2026 value), in ``FLAGS`` order. + + A convention-set value gets ``convention_set`` and nothing else: the + convention, not the parameter's YAML, is then what the reference rests on. + ``category`` matters only for an explicit 2026 entry: an economic series + accepts an index release as its 2026 publication (``supports_2026_value``). + """ + if origin not in ORIGINS: + raise ValueError(f"unknown origin: {origin!r}") + if origin == "convention" or convention_modules: + return ["convention_set"] + flags = set() + if origin in COMPUTED_ORIGINS: + flags.add("computed_or_indexed") + if origin == "explicit_2026" and not any( + supports_2026_value(citation, category) for citation in citations + ): + flags.add("explicit_2026_without_2026_publication") + if comment_evidence and comment_evidence.get("computation"): + flags.add("computed_in_yaml_comment") + if origin == "created_at_load": + flags.add("created_at_load") + if origin == "unresolved": + flags.add("origin_unresolved") + if not any(citation.get("source_class") == "government" for citation in citations): + flags.add("no_government_citation") + statuses = [citation.get("pre_freeze") for citation in citations] + if any(status is False for status in statuses) and not any( + status is True for status in statuses + ): + flags.add("post_freeze_citation") + if origin == "carried_forward": + flags.add("carried_forward") + return [flag for flag in FLAGS if flag in flags] + + +def parameter_category(name: str) -> str: + """``model_switch`` (abolitions, contributed reforms, simulation switches), + ``economic_series`` (price indexes, projections, calibration and uprating + series) or ``law``. + """ + if name.startswith(MODEL_SWITCH_PREFIXES): + return "model_switch" + if name.startswith(ECONOMIC_SERIES_PREFIXES) or ".uprating" in name: + return "economic_series" + return "law" + + +def value_kind(value: Any) -> str: + """``switch`` (a bool), ``list`` (a list of names) or ``number``.""" + if isinstance(value, bool): + return "switch" + if isinstance(value, list): + return "list" + return "number" + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +LOUISIANA_STANDARD_DEDUCTION = "gov.states.la.tax.income.deductions.standard.amount" +# The flags that say the 2026 amount itself may not have been published. +PUBLICATION_FLAGS = ( + "computed_or_indexed", + "explicit_2026_without_2026_publication", + "computed_in_yaml_comment", + "created_at_load", + "origin_unresolved", + "post_freeze_citation", +) + + +YAML_ROOT = "policyengine_us/parameters/" +# Fields an engine fact carries into its report entry. +FACT_FIELDS = ( + "parameter", + "value", + "entry_instant", + "instants", + "origin", + "baseline_origin", + "baseline_value", + "baseline_entry_instant", + "origin_detail", + "convention_modules", + "label", + "unit", + "uprating", + "yaml_file", + "yaml_comments", + "scored_cells", + "unscored_cells", +) + + +def build_entry(fact: Mapping[str, Any]) -> dict[str, Any]: + """One report entry from one engine fact: citations classified, flags set. + + The entry keeps ``FACT_FIELDS`` (the plain-system value and date default to + the reference ones; ``yaml_file`` is relative to ``YAML_ROOT``) and adds + the category, value kind, classified citations, comment evidence, explicit + 2026 support, flags and cell counts. + """ + category = parameter_category(fact["parameter"]) + citations = [ + classify_citation(item.get("title"), item.get("href"), item.get("level", "")) + for item in fact.get("references") or [] + ] + comments = list(fact.get("yaml_comments") or []) + evidence = comment_computation(comments, fact.get("value")) + flags = value_flags( + fact["origin"], + citations, + convention_modules=fact.get("convention_modules") or (), + comment_evidence=evidence, + category=category, + ) + entry = {field: fact.get(field) for field in FACT_FIELDS} + entry["baseline_origin"] = entry["baseline_origin"] or fact["origin"] + if "baseline_value" not in fact: + entry["baseline_value"] = fact.get("value") + if not entry["baseline_entry_instant"]: + entry["baseline_entry_instant"] = fact.get("entry_instant") + detail = dict(entry["origin_detail"] or {}) + detail.pop("yaml_entry", None) # the baseline entry date says the same + entry["origin_detail"] = detail + entry["convention_modules"] = list(entry["convention_modules"] or []) + entry["instants"] = list(entry["instants"] or []) + entry["yaml_comments"] = comments + yaml_file = entry["yaml_file"] + if yaml_file and yaml_file.startswith(YAML_ROOT): + entry["yaml_file"] = yaml_file[len(YAML_ROOT) :] + entry["scored_cells"] = sorted(fact.get("scored_cells") or []) + entry["unscored_cells"] = sorted(fact.get("unscored_cells") or []) + entry["category"] = category + entry["value_kind"] = value_kind(fact.get("value")) + entry["citations"] = citations + entry["comment_evidence"] = evidence + entry["explicit_2026_support"] = ( + explicit_support(citations, category) + if fact["origin"] == "explicit_2026" + else None + ) + entry["flags"] = flags + entry["review"] = any(flag in REVIEW_FLAGS for flag in flags) + entry["publication_review"] = any(flag in PUBLICATION_FLAGS for flag in flags) + entry["scored_cell_count"] = len(entry["scored_cells"]) + entry["unscored_cell_count"] = len(entry["unscored_cells"]) + return entry + + +def _flag_counts(entries: Sequence[Mapping[str, Any]], flag: str) -> dict[str, int]: + hits = [entry for entry in entries if flag in entry["flags"]] + return { + "values": len(hits), + "law_values": sum(entry["category"] == "law" for entry in hits), + "parameters": len({entry["parameter"] for entry in hits}), + "scored_cells": len({cell for entry in hits for cell in entry["scored_cells"]}), + } + + +def summarize(entries: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Counts over the entries read by at least one scored cell.""" + scored = [entry for entry in entries if entry["scored_cell_count"]] + review = [entry for entry in scored if entry["review"]] + publication = [entry for entry in scored if entry["publication_review"]] + unpublished_explicit = [ + entry + for entry in scored + if "explicit_2026_without_2026_publication" in entry["flags"] + ] + return { + "values": len(entries), + "values_read_by_scored_cells": len(scored), + "values_read_only_by_unscored_cells": len(entries) - len(scored), + "parameters_read_by_scored_cells": len({e["parameter"] for e in scored}), + "scored_cells_reading_any_value": len( + {cell for entry in scored for cell in entry["scored_cells"]} + ), + "by_flag": {flag: _flag_counts(scored, flag) for flag in FLAGS}, + "by_origin": dict(sorted(Counter(e["origin"] for e in scored).items())), + "by_category": dict(sorted(Counter(e["category"] for e in scored).items())), + "review_values": len(review), + "review_values_by_category": dict( + sorted(Counter(e["category"] for e in review).items()) + ), + "scored_cells_reading_a_review_value": len( + {cell for entry in review for cell in entry["scored_cells"]} + ), + "publication_review_values": len(publication), + "publication_review_law_values": sum( + entry["category"] == "law" for entry in publication + ), + "scored_cells_reading_a_publication_review_value": len( + {cell for entry in publication for cell in entry["scored_cells"]} + ), + "explicit_2026_without_2026_publication_by_support": { + tier: sum(e["explicit_2026_support"] == tier for e in unpublished_explicit) + for tier in SUPPORT_TIERS + }, + "explicit_2026_without_2026_publication_by_kind": dict( + sorted(Counter(e["value_kind"] for e in unpublished_explicit).items()) + ), + } + + +def _sort_key(entry: Mapping[str, Any]) -> tuple: + return ( + -entry["scored_cell_count"], + entry["parameter"], + entry["entry_instant"] or "", + ) + + +def _format_value(value: Any) -> str: + if isinstance(value, bool) or value is None: + return str(value) + if isinstance(value, (int, float)): + if float(value).is_integer() and abs(value) >= 1: + return f"{value:,.0f}" + return f"{value:.6g}" + text = str(value) + return text if len(text) <= 40 else text[:37] + "..." + + +def _cell(text: Any) -> str: + return str(text).replace("|", "\\|").replace("\n", " ") + + +_DISPLAY_KIND_ORDER = ( + "register", + "agency_publication", + "regulation", + "statute", + "withholding", + "index_data", + "other", +) + + +def _citation_rank(citation: Mapping[str, Any]) -> tuple: + return ( + not citation["dated_2026_publication"], + citation["source_class"] != "government", + _DISPLAY_KIND_ORDER.index(citation["kind"]), + citation["pre_freeze"] is not True, + ) + + +def _citation_label(citation: Mapping[str, Any], width: int = 60) -> str: + label = citation["title"] or citation["host"] or citation["href"] or "" + if len(label) > width: + label = label[: width - 3] + "..." + if citation["publication_date"]: + label += f", {citation['publication_date']['latest']}" + elif citation["applies_to_year"]: + label += f", for {citation['applies_to_year']}" + return f"{citation['kind']}: {label}" + + +def _best_citation(entry: Mapping[str, Any]) -> str: + citations = entry["citations"] + if not citations: + return "none" + best = sorted(citations, key=_citation_rank)[0] + return f"{_citation_label(best)} ({len(citations)} cited)" + + +def _flag_list(flags: Iterable[str], keep: Iterable[str]) -> str: + keep = set(keep) + return ", ".join(flag for flag in flags if flag in keep) + + +def _louisiana_lines(entries: Sequence[Mapping[str, Any]]) -> list[str]: + louisiana = sorted( + ( + entry + for entry in entries + if entry["parameter"].startswith(LOUISIANA_STANDARD_DEDUCTION + ".") + ), + key=lambda entry: entry["parameter"], + ) + lines = ["## Louisiana standard deduction", ""] + if not louisiana: + return lines + ["No cell reads this parameter.", ""] + first = louisiana[0] + lines += [ + f"policyengine-us {first.get('policyengine_us', '2.15.17')} holds " + f"`{LOUISIANA_STANDARD_DEDUCTION}` in `{first.get('yaml_file')}`." + + ( + " The parameter carries `uprating` metadata " + f"(`{first.get('uprating')}`), so years after its last entry are " + "uprated at load." + if first.get("indexed_parameter") + else "" + ), + "", + ] + for entry in louisiana: + cells = entry["scored_cells"] + lines.append( + f"- `{entry['parameter']}` = {_format_value(entry['value'])} from the " + f"entry dated {entry['entry_instant']} (origin `{entry['origin']}`, " + f"support `{entry['explicit_2026_support']}`); flags: " + f"{', '.join(f'`{flag}`' for flag in entry['flags']) or 'none'}; read by " + f"{len(cells)} scored cells" + + ( + f" ({', '.join(cells[:8])}{'...' if len(cells) > 8 else ''})" + if cells + else "" + ) + + "." + ) + evidence = entry.get("comment_evidence") or {} + for comment in evidence.get("comments") or []: + lines.append(f" - YAML comment: {comment}") + for expression in evidence.get("expressions") or []: + lines.append( + f" - `{expression['text']}` = {expression['result']:,.2f}; " + f"reproduces the value: {expression['matches_value']}." + ) + lines += ["", "Its citations, as classified:", ""] + for citation in louisiana[0]["citations"]: + date_text = (citation["publication_date"] or {}).get("latest", "none stated") + lines.append( + f"- {citation['title'] or citation['href']} ({citation['host']}): " + f"{citation['source_class']}, {citation['kind']}; issue date {date_text}; " + f"applies to {citation['applies_to_year'] or 'no stated year'}; " + f"pre-freeze {citation['pre_freeze']}; 2026 publication " + f"{citation['dated_2026_publication']}." + ) + candidates = sum( + supports_2026_value(c, first["category"]) for c in first["citations"] + ) + lines += [ + "", + f"{candidates} of the {len(first['citations'])} citations is a candidate " + "2026 publication under this check's rules. The audit brief records that " + "the Louisiana Department of Revenue published $12,875 (provisional " + "withholding) before the freeze and $12,838 (RIB 26-019) on 2026-09-28; " + "this check reads only metadata and has not read those documents.", + "", + ] + return lines + + +def render_markdown(report: Mapping[str, Any], table_limit: int = 200) -> str: + """The Markdown summary of a report built by ``build_report``.""" + meta = report["meta"] + summary = report["summary"] + entries = report["entries"] + value_check = meta.get("recorded_value_check") or {} + lines = [ + "# Publication sources of 2026 reference parameters", + "", + "For every 2026 parameter value a scored PolicyBench reference reads, this " + "report says where the value comes from (a YAML entry dated in 2026, an " + "older entry carried forward, a value computed while the parameter tree " + "loads, or a reference convention) and whether its citations include a " + "government publication for 2026 not dated after the " + f"{FREEZE_DATE.isoformat()} freeze. Generated by " + "`reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py`" + " with `policybench.publication_sources`; every entry, citation and cell is in " + "`publication_sources.json`.", + "", + f"- Reference system: {meta.get('reference_system')}; policyengine-us " + f"{meta.get('policyengine_us')}, policyengine-core " + f"{meta.get('policyengine_core')}.", + f"- Payload `{meta.get('payload')}`, sha256 `{meta.get('payload_sha256')}`.", + f"- Reproduction: {meta.get('outputs_recomputed')} outputs recomputed " + f"({meta.get('scored_cells_recomputed')} scored); " + f"{meta.get('scored_reference_mismatches')} scored references differ beyond " + "1e-3.", + f"- Recorded leaf values checked against the reference tree: " + f"{value_check.get('checked')}; differing: " + f"{len(value_check.get('mismatches') or [])}.", + f"- Households recomputed in reverse output order: " + f"{meta.get('reverse_order_households')}; outputs whose reads changed: " + f"{len(meta.get('reverse_order_mismatches') or [])}.", + "", + "## Counts", + "", + f"{summary['values_read_by_scored_cells']:,} (parameter, 2026 value) pairs " + f"across {summary['parameters_read_by_scored_cells']:,} parameters are read " + f"by {summary['scored_cells_reading_any_value']:,} scored cells; " + f"{summary['values_read_only_by_unscored_cells']:,} more are read only by " + "unscored cells (listed in the JSON, not counted below). " + f"{summary['publication_review_values']:,} values " + f"({summary['publication_review_law_values']:,} of them law parameters) carry " + "a flag saying the 2026 amount may not have been published before the " + f"freeze; {summary['scored_cells_reading_a_publication_review_value']:,} " + "scored cells read at least one.", + "", + "| Flag | Values | Law values | Parameters | Scored cells |", + "| --- | ---: | ---: | ---: | ---: |", + ] + for flag, counts in summary["by_flag"].items(): + lines.append( + f"| `{flag}` | {counts['values']:,} | {counts['law_values']:,} | " + f"{counts['parameters']:,} | {counts['scored_cells']:,} |" + ) + lines += ["", "| Origin | Values |", "| --- | ---: |"] + for origin, count in summary["by_origin"].items(): + lines.append(f"| `{origin}` | {count:,} |") + tiers = summary["explicit_2026_without_2026_publication_by_support"] + lines += [ + "", + "Explicit 2026 entries without a 2026 publication, by the strongest support " + "their citations do give: " + + ", ".join( + f"{tier} {count:,}" + for tier, count in tiers.items() + if tier != "2026_publication" + ) + + "; by value kind: " + + ", ".join( + f"{kind} {count:,}" + for kind, count in summary[ + "explicit_2026_without_2026_publication_by_kind" + ].items() + ) + + ". `undated_publication` means an agency page that names no date or year " + "(often one listing every year's amounts); `recent_law` a 2025 or 2026 " + "statute or regulation, which may itself state the amount; `inputs_only` " + "withholding guidance or index data, which feed an amount without stating " + "it; `other` none of these.", + "", + ] + lines += _louisiana_lines(entries) + + flagged = sorted( + ( + entry + for entry in entries + if entry["publication_review"] + and entry["scored_cell_count"] + and entry["category"] == "law" + ), + key=_sort_key, + ) + shown = flagged[:table_limit] + lines += [ + "## Law values whose 2026 amount may not have been published", + "", + f"{len(flagged):,} law values read by scored cells carry one of " + + ", ".join(f"`{flag}`" for flag in PUBLICATION_FLAGS) + + f". Ordered by the scored cells reading them; the first {len(shown)} are " + "shown.", + "", + "| # | Parameter | 2026 value | Unit | Entry | Origin | Flags | Support | " + "Scored cells | Best citation |", + "| ---: | --- | ---: | --- | --- | --- | --- | --- | ---: | --- |", + ] + for rank, entry in enumerate(shown, 1): + lines.append( + f"| {rank} | `{_cell(entry['parameter'])}` | " + f"{_cell(_format_value(entry['value']))} | " + f"{_cell(entry.get('unit') or '')} | " + f"{entry['entry_instant']} | {entry['origin']} | " + f"{_flag_list(entry['flags'], PUBLICATION_FLAGS)} | " + f"{entry['explicit_2026_support'] or ''} | {entry['scored_cell_count']} | " + f"{_cell(_best_citation(entry))} |" + ) + + uncited = sorted( + ( + entry + for entry in entries + if entry["scored_cell_count"] + and entry["category"] == "law" + and not entry["publication_review"] + and "no_government_citation" in entry["flags"] + ), + key=_sort_key, + ) + hosts = Counter( + citation["host"] or "(no link)" + for entry in uncited + for citation in entry["citations"] + ) + lines += [ + "", + "## Law values cited to no government source", + "", + f"{len(uncited):,} further law values read by scored cells have no " + "publication flag but cite no government source (most cite a law " + "republisher such as Cornell LII, or nothing). Most cited hosts: " + + ( + ", ".join(f"{host} {count:,}" for host, count in hosts.most_common(8)) + or "none" + ) + + f". The {min(40, len(uncited))} read by the most scored cells:", + "", + "| # | Parameter | 2026 value | Entry | Origin | Scored cells " + "| Best citation |", + "| ---: | --- | ---: | --- | --- | ---: | --- |", + ] + for rank, entry in enumerate(uncited[:40], 1): + lines.append( + f"| {rank} | `{_cell(entry['parameter'])}` | " + f"{_cell(_format_value(entry['value']))} | {entry['entry_instant']} | " + f"{entry['origin']} | {entry['scored_cell_count']} | " + f"{_cell(_best_citation(entry))} |" + ) + + conventions = sorted( + ( + e + for e in entries + if "convention_set" in e["flags"] and e["scored_cell_count"] + ), + key=_sort_key, + ) + modules = Counter( + module + for entry in conventions + for module in entry["convention_modules"] or ["(value differs)"] + ) + other = report.get("other_instants") or [] + lines += [ + "", + "## Convention-set values", + "", + f"{len(conventions):,} values read by scored cells come from `latest_final`, " + "so their own YAML is not what the reference rests on: " + + ( + ", ".join( + f"`{module}` {count:,}" for module, count in sorted(modules.items()) + ) + or "none" + ) + + ".", + "", + "## Economic series and model switches", + "", + "Of the values read by scored cells, " + f"{summary['by_category'].get('economic_series', 0):,} are economic series " + "(price indexes, projections, calibration and uprating series) and " + f"{summary['by_category'].get('model_switch', 0):,} are model " + "switches; they are flagged like any other value in the JSON but left out " + "of the tables above.", + "", + "## Reads at instants outside 2026", + "", + f"{len(other):,} (parameter, instant) reads fall outside 2026 (prior-year " + "lookbacks and similar), " + f"{sum(1 for item in other if item['scored_cells']):,} of them by scored " + "cells. They are listed under `other_instants` in the JSON and not flagged.", + "", + "## What this check can and cannot show", + "", + "- It reads citation metadata, not the cited documents. A 2026 publication " + "among a value's citations is a candidate only: this check does not show " + "that the publication states the value. A flagged value is one with no " + "such candidate.", + "- A statute or regulation citation dates the law, not the 2026 amount, so " + "it never counts as a 2026 publication. Withholding guidance and index " + "releases (CPI) do not count either, except that an index release counts " + "for an economic series.", + "- Dates are what a title or link states. A document that names only the " + "year it applies to is taken to be issued between the year before and the " + "year after, so its freeze status is usually unknown. A date a statute or " + "regulation citation states is usually an effective date, so statute and " + "regulation citations have no freeze status.", + "- `backdated` values come from policyengine-us's `backdate_parameters`, " + "which copies each parameter's earliest value back to 2015; when that " + "earliest entry is dated after a 2026 instant the reference reads, the " + "value read is one the YAML itself dates only from that later entry " + "(`origin_detail.backdated_from`).", + "- Citations are the value's own entry, the parameter's metadata and the " + "nodes defined in the same YAML file; folder `index.yaml` citations are " + "not read.", + "- Reads are recorded inside formula calculation. Parameters read while a " + "simulation is constructed (structural-reform switches) and the " + "`gov.abolitions.*` switches core reads before every variable are not " + "recorded.", + "", + ] + return "\n".join(lines) + + +# Entry fields the JSON leaves out when they hold their default, and how +# ``expand_report`` restores them. +_SPARSE_DEFAULTS = { + "baseline_value": lambda entry: entry["value"], + "baseline_entry_instant": lambda entry: entry["entry_instant"], + "baseline_origin": lambda entry: entry["origin"], + "origin_detail": lambda entry: {}, + "convention_modules": lambda entry: [], + "uprating": lambda entry: None, + "yaml_comments": lambda entry: [], + "comment_evidence": lambda entry: None, + "explicit_2026_support": lambda entry: None, + "label": lambda entry: None, + "unit": lambda entry: None, + "instants": lambda entry: [f"{REFERENCE_YEAR}-01-01"], +} +# Entry fields the JSON leaves out because ``expand_report`` recomputes them. +_DERIVED = { + "category": lambda entry: parameter_category(entry["parameter"]), + "value_kind": lambda entry: value_kind(entry["value"]), + "review": lambda entry: any(flag in REVIEW_FLAGS for flag in entry["flags"]), + "publication_review": lambda entry: any( + flag in PUBLICATION_FLAGS for flag in entry["flags"] + ), + "scored_cell_count": lambda entry: len(entry["scored_cells"]), + "unscored_cell_count": lambda entry: len(entry["unscored_cells"]), +} +ENCODING = ( + "Cells are 'scenario_id/output' strings listed once in 'cells'. " + "'cell_sets' lists sorted index lists into 'cells'; an entry's " + "'scored_cells' and 'unscored_cells' are indexes into 'cell_sets'. " + "'citations' lists each distinct classified citation once and " + "'citation_lists' each distinct list of [index into 'citations', level] " + "pairs; an entry's 'citations' is an index into 'citation_lists'. Entry " + "fields equal to their default are omitted: baseline_value, " + "baseline_entry_instant and baseline_origin default to value, " + "entry_instant and origin; origin_detail to {}; convention_modules and " + "yaml_comments to []; instants to ['2026-01-01']; uprating, " + "comment_evidence, explicit_2026_support, label and unit to null. " + "category, value_kind, review, publication_review and the cell counts are " + "not stored: they follow from parameter, value, flags and cells. " + "yaml_file is relative to meta.yaml_root. " + "policybench.publication_sources.expand_report decodes all of this." +) + + +def _same_json(a: Any, b: Any) -> bool: + return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + +def compact_report( + meta: Mapping[str, Any], + summary: Mapping[str, Any], + entries: Sequence[Mapping[str, Any]], + other_instants: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """The JSON form of a report: cells, cell sets and citations listed once.""" + every_cell = sorted( + { + cell + for item in [*entries, *other_instants] + for key in ("scored_cells", "unscored_cells") + for cell in item[key] + } + ) + cell_index = {cell: index for index, cell in enumerate(every_cell)} + cell_sets: list[list[int]] = [] + set_index: dict[tuple[int, ...], int] = {} + + def cell_set(cells: Iterable[str]) -> int: + key = tuple(sorted(cell_index[cell] for cell in cells)) + if key not in set_index: + set_index[key] = len(cell_sets) + cell_sets.append(list(key)) + return set_index[key] + + citations: list[dict] = [] + citation_index: dict[str, int] = {} + citation_lists: list[list] = [] + list_index: dict[str, int] = {} + + def citation_ref(citation: Mapping[str, Any]) -> list: + body = {key: value for key, value in citation.items() if key != "level"} + key = json.dumps(body, sort_keys=True) + if key not in citation_index: + citation_index[key] = len(citations) + citations.append(body) + return [citation_index[key], citation["level"]] + + def citation_list(items: Sequence[Mapping[str, Any]]) -> int: + refs = [citation_ref(citation) for citation in items] + key = json.dumps(refs) + if key not in list_index: + list_index[key] = len(citation_lists) + citation_lists.append(refs) + return list_index[key] + + compact_entries = [] + for entry in entries: + item = dict(entry) + for field, default in _SPARSE_DEFAULTS.items(): + if field in item and _same_json(item[field], default(entry)): + del item[field] + for field in _DERIVED: + item.pop(field, None) + item["citations"] = citation_list(entry["citations"]) + item["scored_cells"] = cell_set(entry["scored_cells"]) + item["unscored_cells"] = cell_set(entry["unscored_cells"]) + compact_entries.append(item) + compact_other = [ + { + **item, + "scored_cells": cell_set(item["scored_cells"]), + "unscored_cells": cell_set(item["unscored_cells"]), + } + for item in other_instants + ] + return { + "meta": {**dict(meta), "encoding": ENCODING, "yaml_root": YAML_ROOT}, + "summary": dict(summary), + "cells": every_cell, + "cell_sets": cell_sets, + "citations": citations, + "citation_lists": citation_lists, + "entries": compact_entries, + "other_instants": compact_other, + } + + +def expand_report(report: Mapping[str, Any]) -> dict[str, Any]: + """Decode a ``compact_report`` back to entries with cell and citation lists.""" + cells = report["cells"] + sets = [[cells[index] for index in cell_set] for cell_set in report["cell_sets"]] + citations = report["citations"] + lists = report["citation_lists"] + entries = [] + for item in report["entries"]: + entry = dict(item) + entry["citations"] = [ + {**citations[index], "level": level} + for index, level in lists[item["citations"]] + ] + entry["scored_cells"] = list(sets[item["scored_cells"]]) + entry["unscored_cells"] = list(sets[item["unscored_cells"]]) + for field, default in _SPARSE_DEFAULTS.items(): + if field not in entry: + entry[field] = default(entry) + for field, derive in _DERIVED.items(): + entry[field] = derive(entry) + entries.append(entry) + other = [ + { + **item, + "scored_cells": list(sets[item["scored_cells"]]), + "unscored_cells": list(sets[item["unscored_cells"]]), + } + for item in report.get("other_instants") or [] + ] + return { + "meta": dict(report["meta"]), + "summary": dict(report["summary"]), + "entries": entries, + "other_instants": other, + } + + +def dumps_report(report: Mapping[str, Any]) -> str: + """The report as JSON text with one list item per line. + + Valid JSON (``json.loads`` reads it back unchanged), compact within an + item, and line-oriented so a regenerated report diffs entry by entry. + """ + parts = [] + for key in sorted(report): + value = report[key] + if isinstance(value, list): + items = ",\n".join( + json.dumps(item, sort_keys=True, separators=(",", ":")) + for item in value + ) + body = f"[\n{items}\n]" if value else "[]" + else: + body = json.dumps(value, sort_keys=True, indent=1) + parts.append(f"{json.dumps(key)}: {body}") + return "{\n" + ",\n".join(parts) + "\n}\n" + + +def build_report( + facts: Sequence[Mapping[str, Any]], + other_instants: Sequence[Mapping[str, Any]], + meta: Mapping[str, Any], +) -> tuple[dict[str, Any], str]: + """The JSON report (``compact_report`` form) and its Markdown summary.""" + entries = sorted( + (build_entry(fact) for fact in facts), + key=lambda e: (e["parameter"], e["entry_instant"] or ""), + ) + other = sorted( + ( + { + "parameter": item["parameter"], + "instant": item["instant"], + "value": item.get("value"), + "scored_cells": sorted(item.get("scored_cells") or []), + "unscored_cells": sorted(item.get("unscored_cells") or []), + } + for item in other_instants + ), + key=lambda item: (item["parameter"], item["instant"]), + ) + full_meta = { + **dict(meta), + "freeze_date": FREEZE_DATE.isoformat(), + "reference_year": REFERENCE_YEAR, + "flags": list(FLAGS), + "review_flags": list(REVIEW_FLAGS), + "publication_flags": list(PUBLICATION_FLAGS), + "origins": list(ORIGINS), + "support_tiers": list(SUPPORT_TIERS), + } + summary = summarize(entries) + expanded = { + "meta": full_meta, + "summary": summary, + "entries": entries, + "other_instants": other, + } + return compact_report(full_meta, summary, entries, other), render_markdown(expanded) diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py new file mode 100644 index 00000000..2778d00d --- /dev/null +++ b/policybench/reference_adversary.py @@ -0,0 +1,1962 @@ +"""Reference adversary: a law-first judge that attacks consensus-flagged references. + +The diagnosis judge in :mod:`policybench.audit` explains why models missed the +reference and is told to treat the reference as correct. That design cannot +catch a wrong reference unless the judge already knows the contradicting law. +The reference adversary does the opposite job. It takes the cells that +:mod:`policybench.consensus` flags (many models, or several of the strongest, +agreeing on one answer other than the reference) and tries to show from +primary law that the reference is wrong. + +Each case runs in two mechanically separated stages: + +1. Law first. The judge sees the exact household prompt the models saw, the + output's definition, the reference value and the consensus answers with + each member model's explanation, but not how the engine derived the + reference. It works the answer from primary law in force for the tax year, + cites every rule, and says which answer the law supports. It may use web + search and fetch but not PolicyEngine, PolicyBench or their repositories + (:data:`BLOCKED_DOMAINS`). +2. Reconcile. A fresh call sees everything from stage 1, the stage-1 result + verbatim with its sha256, and only now the engine derivation. Stage 1 is + frozen: if the derivation shows stage 1 erred, the judge must say so in + ``stage1_error`` rather than silently change course. It returns a verdict + (:data:`VERDICTS`) and a suggested developer adjudication. + +This module owns the deterministic parts: assembling cases from a frozen +dashboard payload and a consensus-flags report (``build_adversary_cases``), +writing prompts with content-aware resumability (``prepare_adversary``, +``write_stage2_prompt``), validating and publishing judge outputs and their +provenance sidecars (the ``finalize`` command the runners call), and folding +verdicts into a table, a cross-judge comparison and a developer adjudication +queue in the case-notes schema (``collect_adversary``, ``merge_judges``, +``adjudication_queue``, ``apply_adversary_flags``). The runners +``scripts/run_reference_adversary_claude.sh`` and +``scripts/run_reference_adversary_codex.sh`` make the judge calls. + +A verdict never changes a score or a reference. A case whose verdict is not +``reference_holds`` goes to developer adjudication through the existing +``reference_suspect`` flag, which blocks a snapshot freeze until a developer +records a ``reference_verdict`` (:mod:`policybench.adjudications`). Nothing +here reads a score field or writes a prediction. +""" + +from __future__ import annotations + +import argparse +import datetime +import glob +import hashlib +import json +import math +import os +import shutil +import sys +from dataclasses import asdict, dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +from policybench.config import TAX_YEAR + +if TYPE_CHECKING: + import pandas as pd + +# The date the frozen references' law was cut off: the convention module +# reference_audit/2026-09-28/fixes/latest_final.py encodes law published before +# this date. +REFERENCE_FREEZE = "2026-07-03" + +VERDICTS = ( + "reference_holds", + "reference_wrong", + "definition_mismatch", + "prompt_ambiguous", +) +LAW_SUPPORTS = ("reference", "consensus", "neither", "both_readings") +CONFIDENCES = ("high", "medium", "low") +SUGGESTED_ADJUDICATIONS = ( + "affirmed", + "regenerated", + "engine_defect", + "unlisted_input", + "later_law", + "definition_exclusion", + "none", +) +# The suggestions each verdict may carry ("none" goes with any verdict). The +# stage-2 prompt states these pairings; collect_adversary reports a verdict +# that breaks them as inconsistent. +VERDICT_SUGGESTIONS = { + "reference_holds": ("affirmed", "none"), + "reference_wrong": ("engine_defect", "later_law", "regenerated", "none"), + "definition_mismatch": ("definition_exclusion", "regenerated", "none"), + "prompt_ambiguous": ("unlisted_input", "definition_exclusion", "none"), +} +BLOCKED_DOMAINS = ( + "policybench.org", + "www.policybench.org", + "policyengine.org", + "www.policyengine.org", + "github.com", + "raw.githubusercontent.com", +) +# A URL, citation source or search query naming either of these reaches for +# the engine or the benchmark, whatever the host. +BLOCKED_NAMES = ("policyengine", "policybench") +# Strings whose appearance in a codex judge's tool activity (a shell command, +# its output, a search query or an opened URL) means the judge reached for +# engine-derived material: the derivations this module writes, the frozen +# annotations, the dashboard payload, another judge's outputs, or the engine +# and benchmark themselves. Matched case-insensitively. +DERIVATION_MARKERS = ( + "derivations/", + "us_case_reference_explanations", + "case_reference_explanations", + "data.json", + "referenceexplanation", + "stage2_prompt", + "stage1.json", + "verdict.json", + "case_notes.csv", + "audit_row_annotations", + "adjudications.json", + "policybench", + "policyengine", +) +# The judge's tools under the Claude runner (the CLI adds StructuredOutput for +# --json-schema). Claude Code's WebSearch and WebFetch tools. +CLAUDE_ALLOWED_TOOLS = ("WebSearch", "WebFetch") +# Codex event item types that carry no tool activity, and those whose tool +# activity the audit inspects. Any other item type is reported. +CODEX_QUIET_ITEMS = ("agent_message", "reasoning", "todo_list", "error") +CODEX_AUDITED_ITEMS = ("command_execution", "web_search") +RUNNERS = { + "claude": "scripts/run_reference_adversary_claude.sh", + "codex": "scripts/run_reference_adversary_codex.sh", +} +# Each runner's blinding of stage 1: the Claude judge has no file tools; the +# Codex judge runs from an empty directory and its event log is audited. +BLINDING = {"claude": "tools", "codex": "cwd+log-audit"} + +STAGE1_FILE = "stage1.json" +STAGE1_META = "stage1.meta.json" +STAGE1_PROMPT = "stage1_prompt.md" +STAGE2_PROMPT = "stage2_prompt.md" +VERDICT_FILE = "verdict.json" +VERDICT_META = "verdict.meta.json" +STAGE1_SCHEMA_FILE = "schema_stage1.json" +VERDICT_SCHEMA_FILE = "schema_verdict.json" +# Values within this many dollars count as the same answer when collect checks +# a verdict against its case and its stage 1 (the consensus trigger's default +# tolerance). +VALUE_TOLERANCE = 1.0 + + +# --- Output contracts ---------------------------------------------------------- + +CITATION_SCHEMA: dict = { + "type": "object", + "additionalProperties": False, + "properties": { + "source": { + "type": "string", + "description": "The statute, regulation, publication, form or " + "instructions, named as its issuer names it.", + }, + "pinpoint": { + "type": "string", + "description": "Section, subsection, line, table or page.", + }, + "url": {"type": "string", "description": "The URL you read."}, + "published": { + "type": "string", + "description": "Publication or effective date as the source " + "states it (YYYY-MM-DD where known).", + }, + "pre_freeze": { + "type": ["boolean", "null"], + "description": f"True if published before {REFERENCE_FREEZE}, " + "false if after, null if unknown.", + }, + "quote": { + "type": "string", + "description": "A short verbatim quote of the rule relied on.", + }, + }, + "required": ["source", "pinpoint", "url", "published", "pre_freeze", "quote"], +} + +STAGE1_SCHEMA: dict = { + "type": "object", + "additionalProperties": False, + "properties": { + "independent_answer": { + "type": ["number", "null"], + "description": "Your own answer from the law (1 or 0 for an " + "eligibility output); null only if the law leaves it undetermined " + "on the stated facts.", + }, + "computation": { + "type": "string", + "description": "Your step-by-step computation from the stated " + "facts, naming the rule behind each step.", + }, + "citations": {"type": "array", "minItems": 1, "items": CITATION_SCHEMA}, + "law_supports": {"type": "string", "enum": list(LAW_SUPPORTS)}, + "definition_reading": { + "type": "string", + "description": "How you read the output definition for this household.", + }, + "ambiguity": { + "type": "string", + "description": "A second reading the definition or prompt " + 'genuinely admits and the answer it gives; "" if none.', + }, + "confidence": {"type": "string", "enum": list(CONFIDENCES)}, + }, + "required": [ + "independent_answer", + "computation", + "citations", + "law_supports", + "definition_reading", + "ambiguity", + "confidence", + ], +} + +VERDICT_SCHEMA: dict = { + "type": "object", + "additionalProperties": False, + "properties": { + "verdict": {"type": "string", "enum": list(VERDICTS)}, + "summary": { + "type": "string", + "description": "2-4 sentences: what the law gives, what the " + "engine did, and why the verdict follows.", + }, + "independent_answer": {"type": ["number", "null"]}, + "reference_value": {"type": "number"}, + "consensus_value": {"type": ["number", "null"]}, + "engine_step_at_issue": { + "type": "string", + "description": "The derivation step that departs from the law or " + 'the definition; "" when the reference holds.', + }, + "stage1_error": { + "type": "string", + "description": 'Where stage 1 erred and why; "" if stage 1 stands.', + }, + "citations": {"type": "array", "minItems": 1, "items": CITATION_SCHEMA}, + "suggested_adjudication": { + "type": "string", + "enum": list(SUGGESTED_ADJUDICATIONS), + }, + "confidence": {"type": "string", "enum": list(CONFIDENCES)}, + }, + "required": [ + "verdict", + "summary", + "independent_answer", + "reference_value", + "consensus_value", + "engine_step_at_issue", + "stage1_error", + "citations", + "suggested_adjudication", + "confidence", + ], +} + + +def canonical_json(document: Any) -> str: + """The one serialization of a judge output this module writes and hashes.""" + return json.dumps(document, indent=2, sort_keys=True) + + +def sha256_text(text: str) -> str: + """The sha256 of a string's UTF-8 bytes, as lowercase hex.""" + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def blocked_url(url: str) -> bool: + """Whether a URL is on a blocked domain (or a subdomain) or names the engine.""" + text = str(url or "").strip() + if not text: + return False + if any(name in text.lower() for name in BLOCKED_NAMES): + return True + try: + host = urlsplit(text if "//" in text else f"//{text}").hostname or "" + except ValueError: + return False + host = host.lower().rstrip(".") + return any( + host == domain or host.endswith(f".{domain}") for domain in BLOCKED_DOMAINS + ) + + +def blocked_citation_errors(document: Any) -> list[str]: + """Citations that point at a source the adversary may not consult.""" + if not isinstance(document, dict) or not isinstance( + document.get("citations"), list + ): + return [] + errors = [] + for index, citation in enumerate(document["citations"]): + if not isinstance(citation, dict): + continue + url = str(citation.get("url") or "") + source = str(citation.get("source") or "") + if blocked_url(url) or any(name in source.lower() for name in BLOCKED_NAMES): + errors.append( + f"citations/{index}: cites a blocked source ({source!r}, {url!r})" + ) + return errors + + +def output_errors(schema: dict, document: Any) -> list[str]: + """Schema violations of a judge output plus citations of blocked sources.""" + try: + import jsonschema + except ImportError as exc: # pragma: no cover - environment guard + raise RuntimeError( + "judge outputs are validated with jsonschema; install it (uv sync " + "--extra dev) to prepare, collect or finalize adversary cases" + ) from exc + validator = jsonschema.Draft202012Validator(schema) + errors = sorted( + f"{'/'.join(str(p) for p in error.absolute_path) or ''}: {error.message}" + for error in validator.iter_errors(document) + ) + return errors + blocked_citation_errors(document) + + +def _read_output(path: Path, schema: dict) -> tuple[dict | None, str, list[str]]: + """A judge output file: (document if valid, sha256 of its bytes, errors).""" + if not path.is_file(): + return None, "", ["absent"] + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + return None, digest, [f"not JSON: {exc}"] + errors = output_errors(schema, document) + return (None if errors else document), digest, errors + + +def bound_sidecar(output_path: Path, meta_path: Path) -> dict | None: + """The provenance sidecar if it describes the output's current bytes. + + A sidecar whose ``output_sha256`` does not match the output it sits beside + is stale (the output was re-judged or edited) and carries no provenance. + """ + if not output_path.is_file() or not meta_path.is_file(): + return None + try: + meta = json.loads(meta_path.read_text()) + except (OSError, ValueError): + return None + if not isinstance(meta, dict): + return None + if meta.get("output_sha256") != _sha256_file(output_path): + return None + return meta + + +# --- Cases --------------------------------------------------------------------- + + +@dataclass(frozen=True) +class AdversaryCase: + """One consensus-flagged ``(scenario_id, variable)`` cell to attack. + + ``consensus`` holds the triggering clusters, each a dict with the + cluster's ``answer``, ``n_models``, ``n_top``, ``top_models`` and + ``members`` (each member's ``model``, raw ``prediction`` and + ``explanation``). ``derivation`` is the engine's derivation of the + reference; it is shown only in stage 2 and never written to the manifest. + """ + + case_id: str + country: str + scenario_id: str + variable: str + definition: str + household_prompt: str + reference_value: float + consensus: tuple[dict, ...] + models_answered: int + models_exact: int + derivation: str + spec_id: str = "" + metric_type: str = "amount" + state: str = "" + trigger: tuple[str, ...] = () + + def to_manifest_row(self) -> dict: + """The case as a ``cases.jsonl`` row, without the derivation.""" + row = asdict(self) + derivation = row.pop("derivation") + row["consensus"] = [dict(cluster) for cluster in self.consensus] + row["trigger"] = list(self.trigger) + row["derivation_sha256"] = sha256_text(derivation) if derivation else None + return row + + @classmethod + def from_manifest_row(cls, row: dict, derivation: str) -> AdversaryCase: + """Rebuild a case from its manifest row and its derivation file.""" + return cls( + case_id=str(row["case_id"]), + country=str(row["country"]), + scenario_id=str(row["scenario_id"]), + variable=str(row["variable"]), + definition=str(row["definition"]), + household_prompt=str(row["household_prompt"]), + reference_value=float(row["reference_value"]), + consensus=tuple(dict(cluster) for cluster in row["consensus"]), + models_answered=int(row["models_answered"]), + models_exact=int(row["models_exact"]), + derivation=derivation, + spec_id=str(row.get("spec_id", "")), + metric_type=str(row.get("metric_type", "amount")), + state=str(row.get("state", "") or ""), + trigger=tuple(row.get("trigger", ())), + ) + + +def _case_id(country: str, scenario_id: str, variable: str) -> str: + """Filesystem-safe case id; the same rule as ``policybench.audit._case_id``.""" + safe = f"{country}__{scenario_id}__{variable}" + return "".join(c if (c.isalnum() or c in "._-") else "-" for c in safe) + + +def _clean(value: object) -> str: + if value is None or (isinstance(value, float) and math.isnan(value)): + return "" + text = str(value).strip() + return "" if text.lower() == "nan" else text + + +def load_derivations( + annotations_dir: Path | str, country: str = "us" +) -> dict[tuple[str, str], str]: + """Map ``(scenario_id, variable)`` to the frozen engine derivation. + + Reads ``_case_reference_explanations.csv`` (columns country, + scenario_id, variable, reference_value, trace_lines, explanation, error) + from a frozen annotations directory. Rows with an empty explanation are + skipped, so the caller falls back to the payload's ``referenceExplanation`` + for them. + """ + import pandas as pd + + path = Path(annotations_dir) / f"{country}_case_reference_explanations.csv" + if not path.is_file(): + raise FileNotFoundError(f"no reference explanations at {path}") + frame = pd.read_csv(path, dtype=str, keep_default_na=False) + missing = {"scenario_id", "variable", "explanation"} - set(frame.columns) + if missing: + raise ValueError(f"{path}: missing columns {sorted(missing)}") + if "country" in frame.columns: + frame = frame[frame["country"].astype(str) == country] + derivations: dict[tuple[str, str], str] = {} + for row in frame.itertuples(index=False): + text = _clean(row.explanation) + key = (str(row.scenario_id), str(row.variable)) + if text and key not in derivations: + derivations[key] = text + return derivations + + +def _number(value: float) -> str: + """A number as the judge reads it: thousands separators, no float noise.""" + if isinstance(value, bool): + return str(int(value)) + try: + decimal = Decimal(repr(float(value))) + except (InvalidOperation, ValueError, TypeError): + return str(value) + if decimal == decimal.to_integral_value(): + return f"{int(decimal):,}" + return format(decimal, ",") + + +def _truncate(text: str, limit: int) -> str: + if limit < 0 or len(text) <= limit: + return text + return f"{text[:limit]} [... {len(text) - limit} more characters cut]" + + +def _cell_reference(cell: dict) -> float: + references = {json.dumps(entry.get("groundTruth")) for entry in cell.values()} + if len(references) != 1: + raise ValueError("models disagree on groundTruth") + value = next(iter(cell.values())).get("groundTruth") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"no numeric groundTruth: {value!r}") + return float(value) + + +def _payload_derivation(cell: dict) -> str: + for model in sorted(cell): + text = _clean(cell[model].get("referenceExplanation")) + if text: + return text + return "" + + +def build_adversary_cases( + payload: dict, + flags: list[dict], + *, + derivations: dict | None = None, + max_explanation_chars: int = 2000, +) -> list[AdversaryCase]: + """One :class:`AdversaryCase` per consensus flag. + + ``payload`` is a US dashboard payload (``policybench.consensus. + load_us_payload``) and ``flags`` the ``flags`` list of a consensus-flags + report computed from it. From each flagged cell the case takes the exact + household prompt the models saw (``scenarios[sid].prompt.tool``), the + output's definition as the prompt rendered it, the reference value, and + each triggering cluster's members with their raw prediction and + explanation (cut to ``max_explanation_chars``). It reads no score field and + no annotation. ``derivations`` maps ``(scenario_id, variable)`` to the + engine derivation (``load_derivations``); a cell it does not cover falls + back to the payload's ``referenceExplanation``. + + Raises ``ValueError`` when a flag does not match the payload (another + payload's reference or predictions, a missing scenario or model) or when + a cell is flagged twice. + """ + from policybench.prompts import get_variable_description + from policybench.spec import metric_type_for_output, output_group_id + + country = str(payload.get("country") or "us") + scenarios = payload.get("scenarios") or {} + predictions = payload.get("scenarioPredictions") or {} + cases: list[AdversaryCase] = [] + seen: set[tuple[str, str]] = set() + for flag in flags: + scenario_id = str(flag["scenario_id"]) + variable = str(flag["variable"]) + where = f"{scenario_id} {variable}" + if (scenario_id, variable) in seen: + raise ValueError(f"{where}: flagged more than once") + seen.add((scenario_id, variable)) + scenario = scenarios.get(scenario_id) + if not isinstance(scenario, dict): + raise ValueError(f"{where}: scenario missing from the payload") + household_prompt = ((scenario.get("prompt") or {}).get("tool")) or "" + if not household_prompt: + raise ValueError(f"{where}: the payload has no tool prompt") + cell = (predictions.get(scenario_id) or {}).get(variable) + if not cell: + raise ValueError(f"{where}: no predictions in the payload") + try: + reference = _cell_reference(cell) + except ValueError as exc: + raise ValueError(f"{where}: {exc}") from exc + if float(flag["reference"]) != reference: + raise ValueError( + f"{where}: the flag's reference {flag['reference']!r} is not the " + f"payload's {reference!r} (flags from another payload?)" + ) + clusters = [] + for cluster in flag["clusters"]: + members = [] + flagged = cluster.get("predictions") or {} + for model in cluster["models"]: + entry = cell.get(model) + if not isinstance(entry, dict): + raise ValueError(f"{where}: model {model} has no prediction") + prediction = entry.get("prediction") + if model in flagged and ( + not isinstance(prediction, (int, float)) + or float(prediction) != float(flagged[model]) + ): + raise ValueError( + f"{where}: {model}'s prediction {prediction!r} is not the " + f"flag's {flagged[model]!r} (flags from another payload?)" + ) + members.append( + { + "model": str(model), + "prediction": prediction, + "explanation": _truncate( + _clean(entry.get("explanation")), max_explanation_chars + ), + } + ) + clusters.append( + { + "answer": float(cluster["answer"]), + "n_models": int(cluster["n_models"]), + "n_top": int(cluster.get("n_top", 0)), + "top_models": list(cluster.get("top_models", [])), + "members": members, + } + ) + derivation = "" + if derivations is not None: + derivation = _clean(derivations.get((scenario_id, variable))) + if not derivation: + derivation = _payload_derivation(cell) + cases.append( + AdversaryCase( + case_id=_case_id(country, scenario_id, variable), + country=country, + scenario_id=scenario_id, + variable=variable, + definition=get_variable_description(variable, country=country), + household_prompt=household_prompt, + reference_value=reference, + consensus=tuple(clusters), + models_answered=int(flag["models_answered"]), + models_exact=int(flag["models_exact"]), + derivation=derivation, + spec_id=output_group_id(variable), + metric_type=metric_type_for_output(variable), + state=_clean(flag.get("state") or scenario.get("state")), + trigger=tuple(str(t) for t in flag.get("trigger", ())), + ) + ) + return cases + + +# --- Prompts ------------------------------------------------------------------- + +_DOMAINS = ", ".join(BLOCKED_DOMAINS) + +_SOURCE_RULES = f"""\ +Do not consult PolicyEngine or PolicyBench in any form: not policyengine.org, \ +policybench.org, their GitHub repositories, their documentation, their \ +package source, or any calculator built on them. Do not fetch anything from \ +these domains: {_DOMAINS}. Do not rely on any calculator or estimate built by \ +an AI model. A citation of any of these sources voids your answer.""" + +_STAGE1_HEADER = f"""\ +You are a reference adversary for a US tax-and-benefit benchmark. Each \ +benchmark question gives a household and asks for policy quantities for tax \ +year {TAX_YEAR}. A microsimulation engine produced the reference answer. On \ +the question below, a cluster of AI models answering from memory, without \ +tools, agreed on an answer other than the reference. Agreement among models \ +proves nothing, and neither does the engine. Your job is to find out what the \ +law gives. + +This is stage 1 of 2. In this stage you work the answer out yourself from \ +primary law, before you see anything about how the engine computed its value. + +How to work: +1. Use web search and web fetch to find and read primary law in force for tax \ +year {TAX_YEAR}: statutes, regulations, official agency publications, and \ +official forms and their instructions. Prefer the issuing government's own \ +site. +2. Apply the household prompt's conventions exactly as the models were told \ +them: treat any unlisted numeric input as 0 and any other unlisted fact, \ +boolean or status as false; assume tax filing and program take-up when \ +required; do not infer unlisted income, expenses, assets, benefit receipt, \ +rent or health coverage. +3. Read the output definition literally. It decides which people, tax units, \ +returns, taxes, credits or benefits the number covers. Where it lists \ +components, decide for each candidate amount whether the definition includes \ +it. Where the household holds more than one tax unit or return, decide from \ +the definition which of them the number covers. +4. Cite every rule you rely on: the source, a pinpoint (section, line, table \ +or page), the URL you read, the publication or effective date, and a short \ +verbatim quote. The reference's law was frozen on {REFERENCE_FREEZE}: set \ +pre_freeze to true if the source was published before that date, false if \ +after, and null if you cannot tell. If an amount for {TAX_YEAR} (a standard \ +deduction, bracket, threshold, rate base or allotment) had not been published \ +before {REFERENCE_FREEZE}, say so in computation and say what had been \ +published. +5. {_SOURCE_RULES} +6. Say which answer the law supports: "reference", "consensus", "neither", or \ +"both_readings" (the reference under one reasonable reading of the definition \ +or the facts, the consensus under another). +7. In definition_reading, say how you read the output definition for this \ +household. In ambiguity, describe any second reading that the definition or \ +the household prompt genuinely admits, and the answer it gives; use "" if \ +there is none. +8. independent_answer is your own number (1 or 0 for an eligibility output). \ +Use null only when the stated facts and the law leave it genuinely \ +undetermined, and say why in ambiguity. + +Some output definitions mention PolicyEngine (for example "eligible for \ +Medicaid under PolicyEngine rules"). Work those from the law as well; if the \ +mention could change the answer, say so in ambiguity. + +Return only the JSON object the schema asks for.""" + +_STAGE2_HEADER = f"""\ +You are a reference adversary for a US tax-and-benefit benchmark. Each \ +benchmark question gives a household and asks for policy quantities for tax \ +year {TAX_YEAR}. A microsimulation engine produced the reference answer. On \ +the question below, a cluster of AI models answering from memory, without \ +tools, agreed on an answer other than the reference. + +This is stage 2 of 2: reconciliation. In stage 1, a judge who had not seen \ +how the engine derived the reference worked the question from primary law. \ +Its result appears below exactly as it was recorded, with its sha256. Stage 1 \ +is frozen and you may not revise it. If the engine derivation or the law \ +shows that stage 1 erred (it misread a fact, missed or misapplied a rule, or \ +used the wrong year's amount), say so in stage1_error and name the error. Do \ +not silently change course: if your independent_answer, or your view of which \ +answer the law supports, differs from stage 1's, stage1_error must say why. \ +Use "" for stage1_error only when stage 1 stands. + +Only now do you see how the engine derived the reference. The derivation is a \ +short narrative that a language model wrote from the engine's computation \ +trace for this household; it can misdescribe a step, but the reference value \ +is the engine's own output. Compare each step with the law and with the \ +output definition, and decide: +- reference_holds: the reference is what the law and the output definition \ +give on the stated facts. +- reference_wrong: the engine misapplies the law on the stated facts, or \ +uses an amount or rule the law had not published (for example a {TAX_YEAR} \ +amount the engine projected itself where no agency had published one before \ +{REFERENCE_FREEZE}). +- definition_mismatch: the engine computes something other than what the \ +output definition describes (it includes or leaves out people, returns, \ +taxes, credits or benefits that the definition covers or excludes). +- prompt_ambiguous: the stated facts or the definition genuinely admit more \ +than one answer (for example the answer turns on an input the prompt does not \ +list). + +engine_step_at_issue names the derivation step that departs from the law or \ +the definition; use "" when the reference holds. + +suggested_adjudication: +- affirmed: the reference holds. +- regenerated: the reference should be recomputed under a stated convention \ +or a corrected input; the output stays scored. +- engine_defect: the engine misapplies the law on the stated facts. +- unlisted_input: the answer turns on an input the prompt does not list. +- later_law: the reference rests on law or an amount published after \ +{REFERENCE_FREEZE}, or never published. +- definition_exclusion: the engine's quantity does not match the output \ +definition, so the output should be excluded. +- none: you suggest nothing. +Pair them this way: reference_holds with affirmed; reference_wrong with \ +engine_defect, later_law or regenerated; definition_mismatch with \ +definition_exclusion or regenerated; prompt_ambiguous with unlisted_input or \ +definition_exclusion. "none" goes with any verdict. + +reference_value is the engine reference below. consensus_value is the \ +consensus answer you weighed (null if none applies). + +Cite the law you rely on as in stage 1: source, pinpoint, URL, date, \ +pre_freeze and a short quote. The engine derivation is not a citation. \ +{_SOURCE_RULES} + +Your verdict changes no score. A developer adjudicates every case you do not \ +return as reference_holds. + +Return only the JSON object the schema asks for.""" + + +def _block(title: str, text: str) -> str: + return f"----- BEGIN {title} -----\n{text}\n----- END {title} -----" + + +def _reference_text(case: AdversaryCase) -> str: + value = case.reference_value + if case.metric_type == "binary": + return f"{_number(value)} ({'eligible' if value else 'not eligible'})" + rounded = f"${value:,.2f}" + if round(value, 2) == value: + return rounded + return f"{rounded} (engine output {_number(value)})" + + +def _case_sections(case: AdversaryCase) -> list[str]: + """The case as both stages show it; never the derivation.""" + kind = ( + "an eligibility flag, 1 or 0" + if case.metric_type == "binary" + else "an annual dollar amount" + ) + lines = [ + f"TAX YEAR: {TAX_YEAR} REFERENCE LAW FROZEN: {REFERENCE_FREEZE}", + f"STATE: {case.state or 'see the household prompt'}", + f"OUTPUT: {case.variable} ({kind})", + f"OUTPUT DEFINITION, as the models saw it: {case.definition}", + "", + "HOUSEHOLD PROMPT SHOWN TO THE MODELS (verbatim; it asks for several " + f"outputs, and this case concerns only {case.variable}):", + _block("HOUSEHOLD PROMPT", case.household_prompt), + "", + f"ENGINE REFERENCE VALUE: {_reference_text(case)}", + f"MODELS: {case.models_answered} gave a usable answer; " + f"{case.models_exact} matched the reference.", + "", + "CONSENSUS ANSWERS (clusters of models agreeing on a value other than " + "the reference):", + ] + for number, cluster in enumerate(case.consensus, 1): + lines.append( + f"\nCluster {number}: {_number(cluster['answer'])}, given by " + f"{cluster['n_models']} models ({cluster.get('n_top', 0)} of them " + "among the benchmark's top-ranked models)" + ) + for member in cluster["members"]: + explanation = member.get("explanation") or "(no explanation given)" + lines.append( + f"- {member['model']}: answered {_number(member['prediction'])}\n" + f" explanation: {explanation}" + ) + return lines + + +def render_stage1_prompt(case: AdversaryCase) -> str: + """The law-first prompt: everything about the case except the derivation.""" + return "\n".join([_STAGE1_HEADER, "", *_case_sections(case)]) + + +def render_stage2_prompt(case: AdversaryCase, stage1: dict, stage1_sha256: str) -> str: + """The reconciling prompt: the case, frozen stage 1, then the derivation. + + ``stage1`` is embedded as :func:`canonical_json`, the form the runners + publish ``stage1.json`` in, so the embedded text is the stage-1 file and + ``stage1_sha256`` is the sha256 of exactly that text. + """ + derivation = case.derivation or "(no derivation was recorded for this case)" + return "\n".join( + [ + _STAGE2_HEADER, + "", + *_case_sections(case), + "", + f"STAGE 1 RESULT (frozen; sha256 {stage1_sha256}):", + _block("STAGE 1 RESULT", canonical_json(stage1)), + "", + "ENGINE DERIVATION OF THE REFERENCE:", + _block("ENGINE DERIVATION", derivation), + ] + ) + + +# --- Preparation and resumability --------------------------------------------- + +_STAGE1_OUTPUTS = (STAGE1_FILE, STAGE1_META) +_STAGE2_OUTPUTS = (STAGE2_PROMPT, VERDICT_FILE, VERDICT_META) + + +def _unlink(case_dir: Path, names: tuple[str, ...]) -> None: + for name in names: + (case_dir / name).unlink(missing_ok=True) + + +def prepare_adversary( + adversary_dir: Path | str, cases: list[AdversaryCase] +) -> list[AdversaryCase]: + """Write the schemas, the manifest, stage-1 prompts and derivations. + + Layout under ``adversary_dir``:: + + schema_stage1.json + schema_verdict.json + cases.jsonl (manifest; no derivation) + derivations/.md (stage-2 input, outside the case dir) + cases//stage1_prompt.md + cases//stage1.json (written later by a runner) + cases//stage2_prompt.md (written by write_stage2_prompt) + cases//verdict.json (written later by a runner) + + Resumable and content-aware: a case whose stage-1 prompt changed loses its + stage-1 output, stage-2 prompt, verdict and their sidecars; a case whose + derivation changed loses its stage-2 prompt and verdict but keeps stage 1, + which never saw the derivation. Case directories and derivations of cases + no longer listed are removed. + """ + adversary_dir = Path(adversary_dir) + ids = [case.case_id for case in cases] + duplicates = sorted({case_id for case_id in ids if ids.count(case_id) > 1}) + if duplicates: + raise ValueError(f"duplicate adversary cases: {duplicates}") + adversary_dir.mkdir(parents=True, exist_ok=True) + (adversary_dir / STAGE1_SCHEMA_FILE).write_text( + json.dumps(STAGE1_SCHEMA, indent=2) + "\n" + ) + (adversary_dir / VERDICT_SCHEMA_FILE).write_text( + json.dumps(VERDICT_SCHEMA, indent=2) + "\n" + ) + cases_root = adversary_dir / "cases" + derivations_root = adversary_dir / "derivations" + cases_root.mkdir(exist_ok=True) + derivations_root.mkdir(exist_ok=True) + current = set(ids) + for child in cases_root.iterdir(): + if child.is_dir() and child.name not in current: + shutil.rmtree(child) + for child in derivations_root.iterdir(): + if child.is_file() and child.suffix == ".md" and child.stem not in current: + child.unlink() + with (adversary_dir / "cases.jsonl").open("w") as manifest: + for case in cases: + manifest.write(json.dumps(case.to_manifest_row()) + "\n") + case_dir = cases_root / case.case_id + case_dir.mkdir(exist_ok=True) + prompt_path = case_dir / STAGE1_PROMPT + new_prompt = render_stage1_prompt(case) + if not prompt_path.is_file() or prompt_path.read_text() != new_prompt: + # Stage 1 (and everything built on it) answered another prompt. + _unlink(case_dir, _STAGE1_OUTPUTS + _STAGE2_OUTPUTS) + prompt_path.write_text(new_prompt) + derivation_path = derivations_root / f"{case.case_id}.md" + if ( + not derivation_path.is_file() + or derivation_path.read_text() != case.derivation + ): + # Stage 2 reconciled against another derivation; stage 1 never + # saw it and stands. + _unlink(case_dir, _STAGE2_OUTPUTS) + derivation_path.write_text(case.derivation) + return cases + + +def _load_manifest(adversary_dir: Path) -> dict[str, dict]: + rows: dict[str, dict] = {} + manifest = Path(adversary_dir) / "cases.jsonl" + if manifest.is_file(): + for line in manifest.read_text().splitlines(): + if line.strip(): + row = json.loads(line) + rows[str(row["case_id"])] = row + return rows + + +def load_case(adversary_dir: Path | str, case_id: str) -> AdversaryCase: + """A prepared case with its derivation, from the manifest and derivations/.""" + adversary_dir = Path(adversary_dir) + row = _load_manifest(adversary_dir).get(case_id) + if row is None: + raise ValueError(f"{adversary_dir}/cases.jsonl lists no case {case_id!r}") + derivation_path = adversary_dir / "derivations" / f"{case_id}.md" + derivation = derivation_path.read_text() if derivation_path.is_file() else "" + return AdversaryCase.from_manifest_row(row, derivation) + + +def write_stage2_prompt(adversary_dir: Path | str, case_id: str) -> Path: + """Write ``cases//stage2_prompt.md`` from a valid stage 1. + + Raises ``ValueError`` unless ``stage1.json`` satisfies + :data:`STAGE1_SCHEMA`, cites no blocked source and is in the canonical + form the runners publish (so the embedded text is the file, byte for + byte, and the embedded sha256 is the file's). The stage-2 prompt embeds + stage 1 and its sha256, so a re-judged stage 1 changes it. A + verdict judged on another stage-2 prompt is deleted with its sidecar: one + is kept only if the stored prompt is unchanged, or if its bound sidecar + records the new prompt's sha256. + """ + adversary_dir = Path(adversary_dir) + case = load_case(adversary_dir, case_id) + case_dir = adversary_dir / "cases" / case_id + stage1, stage1_sha, errors = _read_output(case_dir / STAGE1_FILE, STAGE1_SCHEMA) + if stage1 is None: + raise ValueError(f"{case_id}: no valid stage 1 ({'; '.join(errors)})") + if sha256_text(canonical_json(stage1)) != stage1_sha: + raise ValueError( + f"{case_id}: stage1.json is not in canonical form, so no runner " + "published it; re-run stage 1" + ) + prompt = render_stage2_prompt(case, stage1, stage1_sha) + path = case_dir / STAGE2_PROMPT + unchanged = path.is_file() and path.read_text() == prompt + if not unchanged: + meta = bound_sidecar(case_dir / VERDICT_FILE, case_dir / VERDICT_META) + if not (meta and meta.get("prompt_sha256") == sha256_text(prompt)): + _unlink(case_dir, (VERDICT_FILE, VERDICT_META)) + path.write_text(prompt) + return path + + +# --- Collection ---------------------------------------------------------------- + +_VERDICT_COLUMNS = [ + "case_id", + "country", + "scenario_id", + "variable", + "verdict", + "suggested_adjudication", + "confidence", + "reference_value", + "consensus_value", + "independent_answer", + "stage1_law_supports", + "engine_step_at_issue", + "stage1_error", + "summary", + "citations", + "judge_model", +] + + +def _differs(a: Any, b: Any) -> bool: + """Whether two answers (numbers or null) differ beyond the tolerance.""" + if a is None or b is None: + return (a is None) != (b is None) + return abs(float(a) - float(b)) > VALUE_TOLERANCE + + +def verdict_problems(stage1: dict, verdict: dict, case_row: dict) -> list[str]: + """Why a schema-valid verdict does not hang together with its case. + + The checks: the suggestion fits the verdict (:data:`VERDICT_SUGGESTIONS`); + an engine step is named exactly when the reference is wrong or mismatched + (and never when it holds); the verdict departs from stage 1's answer or + finding only with a ``stage1_error``; its ``reference_value`` is the + case's; its ``consensus_value`` is one of the case's consensus answers. + """ + problems = [] + name = verdict["verdict"] + suggested = verdict["suggested_adjudication"] + stage1_error = verdict["stage1_error"].strip() + step = verdict["engine_step_at_issue"].strip() + if suggested not in VERDICT_SUGGESTIONS[name]: + problems.append(f"{name} with suggested_adjudication {suggested}") + if name == "reference_holds" and step: + problems.append("reference_holds names an engine step at issue") + if name in ("reference_wrong", "definition_mismatch") and not step: + problems.append(f"{name} names no engine step at issue") + if not stage1_error: + supports = stage1["law_supports"] + if supports == "consensus" and name == "reference_holds": + problems.append( + "stage 1 found the law supports the consensus; the verdict holds " + "the reference without naming a stage-1 error" + ) + if supports == "reference" and name == "reference_wrong": + problems.append( + "stage 1 found the law supports the reference; the verdict calls " + "it wrong without naming a stage-1 error" + ) + if _differs(stage1["independent_answer"], verdict["independent_answer"]): + problems.append( + f"the verdict's independent answer ({verdict['independent_answer']}) " + f"departs from stage 1's ({stage1['independent_answer']}) without " + "naming a stage-1 error" + ) + reference = float(case_row["reference_value"]) + if _differs(verdict["reference_value"], reference): + problems.append( + f"the verdict's reference_value {verdict['reference_value']} is not the " + f"case's reference {reference}" + ) + consensus_value = verdict["consensus_value"] + answers = [float(cluster["answer"]) for cluster in case_row.get("consensus", [])] + if consensus_value is not None and answers: + if all(_differs(consensus_value, answer) for answer in answers): + problems.append( + f"consensus_value {consensus_value} matches no consensus cluster " + f"({', '.join(_number(a) for a in answers)})" + ) + return problems + + +def _judge_model(meta: dict | None) -> str: + if not meta: + return "unknown" + reported = meta.get("judge_model_reported") or [] + if isinstance(reported, str): + reported = [reported] + if reported: + return "+".join(str(model) for model in reported) + requested = meta.get("judge_model_requested") + return str(requested) if requested else "unknown" + + +def collect_adversary(adversary_dir: Path | str) -> dict[str, pd.DataFrame]: + """Fold the judges' outputs into tables; changes no file. + + Returns ``{"verdicts": ..., "missing": ..., "inconsistent": ...}``. + ``verdicts`` has one row per case with a valid verdict built on a valid + stage 1; ``citations`` is the verdict's citations as a JSON string and + ``judge_model`` comes from the verdict's bound sidecar ("unknown" + without one). ``missing`` lists cases with no such verdict and why: no + verdict, an invalid one, no valid stage 1, or a verdict whose sidecar or + stage-2 prompt was built on another stage 1. ``inconsistent`` lists one + row per problem :func:`verdict_problems` finds, and a stage 1 and verdict + from different runners; those cases stay in ``verdicts`` too. + """ + import pandas as pd + + adversary_dir = Path(adversary_dir) + manifest = _load_manifest(adversary_dir) + cases_root = adversary_dir / "cases" + rows: list[dict] = [] + missing: list[dict] = [] + inconsistent: list[dict] = [] + for case_id, row in manifest.items(): + case_dir = cases_root / case_id + key = { + "case_id": case_id, + "scenario_id": row["scenario_id"], + "variable": row["variable"], + } + verdict, _, verdict_errors = _read_output( + case_dir / VERDICT_FILE, VERDICT_SCHEMA + ) + stage1, stage1_sha, stage1_errors = _read_output( + case_dir / STAGE1_FILE, STAGE1_SCHEMA + ) + if verdict_errors == ["absent"]: + missing.append({**key, "reason": "no verdict"}) + continue + if verdict is None: + missing.append( + {**key, "reason": f"invalid verdict: {'; '.join(verdict_errors)}"} + ) + continue + if stage1 is None: + missing.append( + { + **key, + "reason": "the verdict has no valid stage 1: " + + "; ".join(stage1_errors), + } + ) + continue + meta = bound_sidecar(case_dir / VERDICT_FILE, case_dir / VERDICT_META) + if meta and meta.get("stage1_sha256") not in (None, stage1_sha): + missing.append( + {**key, "reason": "the verdict was judged on another stage 1"} + ) + continue + stage2_prompt = case_dir / STAGE2_PROMPT + if stage2_prompt.is_file() and stage1_sha not in stage2_prompt.read_text(): + missing.append( + { + **key, + "reason": "the stage-2 prompt was built on another stage 1", + } + ) + continue + for problem in verdict_problems(stage1, verdict, row): + inconsistent.append({**key, "problem": problem}) + stage1_meta = bound_sidecar(case_dir / STAGE1_FILE, case_dir / STAGE1_META) + if ( + meta + and stage1_meta + and meta.get("judge_runner") != stage1_meta.get("judge_runner") + ): + inconsistent.append( + { + **key, + "problem": "stage 1 and the verdict come from different runners " + f"({stage1_meta.get('judge_runner')}, {meta.get('judge_runner')})", + } + ) + rows.append( + { + "case_id": case_id, + "country": row["country"], + "scenario_id": row["scenario_id"], + "variable": row["variable"], + "verdict": verdict["verdict"], + "suggested_adjudication": verdict["suggested_adjudication"], + "confidence": verdict["confidence"], + "reference_value": float(row["reference_value"]), + "consensus_value": verdict["consensus_value"], + "independent_answer": verdict["independent_answer"], + "stage1_law_supports": stage1["law_supports"], + "engine_step_at_issue": verdict["engine_step_at_issue"], + "stage1_error": verdict["stage1_error"], + "summary": verdict["summary"], + "citations": json.dumps(verdict["citations"], sort_keys=True), + "judge_model": _judge_model(meta), + } + ) + return { + "verdicts": pd.DataFrame(rows, columns=_VERDICT_COLUMNS), + "missing": pd.DataFrame( + missing, columns=["case_id", "scenario_id", "variable", "reason"] + ), + "inconsistent": pd.DataFrame( + inconsistent, columns=["case_id", "scenario_id", "variable", "problem"] + ), + } + + +_MERGED_FIELDS = ( + "verdict", + "suggested_adjudication", + "confidence", + "independent_answer", + "consensus_value", + "stage1_law_supports", + "engine_step_at_issue", + "stage1_error", + "summary", + "judge_model", +) +_CASE_FIELDS = ("case_id", "country", "scenario_id", "variable", "reference_value") + + +def merge_judges(verdicts_by_judge: dict[str, pd.DataFrame]) -> pd.DataFrame: + """One row per case across judges, keyed by each judge's label. + + Each judge contributes ``