From 1024f06c100dcc8232ec09b47885998971f3d2a6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 06:03:06 -0700 Subject: [PATCH 1/5] Let the childcare lane run on policyengine.py 6.11.0's certified pair Adds a certified pair registry (data/uk/certified_pairs.json, pipeline/uk_certified_pairs.py) for policyengine.py releases the lane runs on before the release-replay pipeline qualifies them. A pair is recorded in the replay variant's identity shape under the id pair- and passes the same identity checks (uk_replay_variants.validate_identity, split out of validate_record), but it is never a replay variant: no layout, no result root, and nothing writes the variant index. The first pair is policyengine.py 6.11.0 with policyengine-uk 2.132.6 and Microcosm build microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71, pinned by a requirements file whose 82 pins the local venv matches. The lane now reads each child's own cap on funded hours (max_free_entitlement_hours_used) as a dataset input, records it in the artifact, and treats it as part of the full-use evidence. The 19 June pair's verified entry states that the cap is not in its dataset. year_block refuses a world whose benefit units weigh nothing. Refs #228. No numbers change: the run on the new pair is still to come. Co-Authored-By: Claude Opus 5.5 --- data/uk/certified_pairs.json | 97 ++++++++ ...-25-national-20261019T083323Z-6cba5d71.txt | 85 +++++++ pipeline/compute_uk_childcare_100k.py | 48 ++-- pipeline/compute_uk_event.py | 9 + pipeline/uk_certified_pairs.py | 140 ++++++++++++ pipeline/uk_replay_variants.py | 103 +++++---- .../construction_annotations.json | 3 +- scorecard_db/ingest_uk_childcare_100k.py | 82 +++++-- tests/test_uk_certified_pairs.py | 212 ++++++++++++++++++ tests/test_uk_childcare_100k.py | 127 +++++++++-- 10 files changed, 816 insertions(+), 90 deletions(-) create mode 100644 data/uk/certified_pairs.json create mode 100644 docs/uk_replay/requirements/pairs/pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.txt create mode 100644 pipeline/uk_certified_pairs.py create mode 100644 tests/test_uk_certified_pairs.py diff --git a/data/uk/certified_pairs.json b/data/uk/certified_pairs.json new file mode 100644 index 000000000..d77593cc1 --- /dev/null +++ b/data/uk/certified_pairs.json @@ -0,0 +1,97 @@ +{ + "_what": "Certified pairs a lane runs on outside the replay variant index (pipeline/uk_certified_pairs.py). Each is one policyengine.py release's certified bundle, in the replay variant's identity shape, under the id pair-. Never a replay variant: no event replay runs on one.", + "kind": "uk_certified_pairs", + "pairs": [ + { + "certification": { + "certified_for_model_version": "2.132.6", + "compatibility_basis": "built_with_model_package", + "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "source": "policyengine.py 6.11.0 packaged bundle manifest (policyengine/data/bundle/manifest.json, sha256 bccf23acde631aed4665757094c04f86e60993fb9d388e7dc958b90c876e5a80; PyPI wheel sha256 b2aa90b97a5c608f4aaad5e6833c21014d786bfd96bb831c9c51b9bb60ff2a50); the pair was certified in PolicyEngine/policyengine.py#685 (merged 2026-10-21)" + }, + "data_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030, + 2031, + 2032 + ], + "data_years_evidence": { + "simulation_dataset_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030, + 2031, + 2032 + ], + "source": "Measured for the same policyengine-uk 2.132.6, policyengine-core 3.33.3 and dataset by the experiment population probe under policyengine 6.10.0 (data/uk/experiments/probes/exp-pe6.10.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.json); every lane run checks its managed simulation's sim.dataset.years against this list before any calculation." + }, + "executor": [ + "local" + ], + "identity": { + "dataset": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "size_bytes": 321798995 + }, + "materialization": "policyengine_data_dir", + "packaged_bundle_manifest_sha256": "bccf23acde631aed4665757094c04f86e60993fb9d388e7dc958b90c876e5a80", + "packages": { + "policyengine": { + "record_sha256": "113897bc23a82dab3fd29a0715c7a111fbb948f9b791eaf866e1324d75c06bbf", + "version": "6.11.0" + }, + "policyengine-core": { + "record_sha256": "e68eb6a81f472df42aff4fbf56c8537d1185075078ffb2e0a6399f1df0b66bc2", + "version": "3.33.3" + }, + "policyengine-uk": { + "record_sha256": "ca7c36b7617f950f48a04e2816a29e1eb962c44097f3ef08ed4036d7414d2709", + "version": "2.132.6" + } + }, + "python": "3.12", + "release_bundle": { + "bundle_id": "uk-6.11.0", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "certified_data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "compatibility_basis": "built_with_model_package", + "country_id": "uk", + "data_build_fingerprint": null, + "data_build_model_git_sha": null, + "data_build_model_version": "2.132.6", + "data_package": "microcosm-data", + "data_version": "0.1.0", + "default_dataset": "microcosm_uk_2024_25", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "model_package": "policyengine-uk", + "model_version": "2.132.6", + "policyengine_version": "6.11.0" + }, + "requirements_sha256": "f57a4899ec639cd89ae0eba15525f1d0bba25ff9bc091c95a14ae4d46d363d80" + }, + "identity_sha256": "9bb966d62e65c29e44a83620b21d720934fb33e57b8fc13695254657acd9b35a", + "note": "policyengine.py 6.11.0 (released 25 October 2026) with the pair its bundle certifies: policyengine-uk 2.132.6 and the Microcosm UK 2024-25 national cut built with it. A lane runs on it before the release-replay pipeline qualifies it; no event replay runs on it.", + "pair_id": "pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "requirements": { + "path": "docs/uk_replay/requirements/pairs/pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.txt", + "sha256": "f57a4899ec639cd89ae0eba15525f1d0bba25ff9bc091c95a14ae4d46d363d80" + }, + "status": "certified_pair" + } + ], + "schema_version": 1 +} diff --git a/docs/uk_replay/requirements/pairs/pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.txt b/docs/uk_replay/requirements/pairs/pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.txt new file mode 100644 index 000000000..f08f27c88 --- /dev/null +++ b/docs/uk_replay/requirements/pairs/pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.txt @@ -0,0 +1,85 @@ +# Certified pair pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71 (data/uk/certified_pairs.json, pipeline/uk_certified_pairs.py); not a replay variant. +# Resolved with uv pip compile for Python 3.12, x86_64-manylinux_2_28, within the exp-pe6.10.0 experiment population's freeze; identical to it apart from the policyengine pin. +# policyengine 6.11.0 from PyPI (policyengine-6.11.0-py3-none-any.whl, sha256 b2aa90b97a5c608f4aaad5e6833c21014d786bfd96bb831c9c51b9bb60ff2a50). +annotated-types==0.8.0 +anyio==4.15.1 +asttokens==3.0.2 +attrs==26.1.0 +blosc2==4.14.1 +certifi==2026.7.22 +charset-normalizer==3.5.2 +click==8.5.0 +decorator==5.3.1 +dpath==2.2.0 +executing==2.3.0 +filelock==4.1.0 +fsspec==2026.9.0 +h11==0.16.0 +h2==4.4.1 +h5py==3.16.0 +hf-xet==1.7.0 +hpack==4.2.0 +httpcore==1.0.9 +httpcore2==2.13.1 +httpx==0.28.1 +httpx2==2.13.1 +huggingface-hub==2.2.0 +hyperframe==6.1.0 +idna==3.20 +iniconfig==2.3.1 +ipython==8.39.0 +jedi==0.20.0 +jinja2==3.1.6 +jsonpickle==4.1.3 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +markdown-it-py==4.2.0 +markupsafe==3.0.4 +matplotlib-inline==0.2.2 +mdurl==0.1.2 +microdf-python==1.3.10 +msgpack==1.2.3 +ndindex==1.10.1 +networkx==3.7 +numexpr==2.14.2 +numpy==2.5.4 +packaging==26.3 +pandas==3.0.6 +parso==0.8.7 +pexpect==4.9.0 +plotly==5.24.1 +pluggy==1.6.0 +policyengine==6.11.0 +policyengine-core==3.33.3 +policyengine-uk==2.132.6 +prompt-toolkit==3.0.53 +psutil==6.1.1 +ptyprocess==0.7.0 +pure-eval==0.2.4 +py-cpuinfo==9.0.0 +pydantic==2.14.0 +pydantic-core==2.50.0 +pygments==2.21.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +pyvis==0.3.2 +pyyaml==6.0.3 +referencing==0.37.0 +requests==2.34.2 +rich==15.0.0 +rpds-py==2026.9.1 +six==1.17.0 +sortedcontainers==2.4.0 +stack-data==0.6.3 +standard-imghdr==3.13.0 +tables==3.11.1 +tenacity==9.2.1 +threadpoolctl==3.7.0 +tqdm==4.70.1 +traitlets==5.16.1 +truststore==0.10.4 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.8.0 +wcwidth==0.9.2 +wheel==0.48.0 diff --git a/pipeline/compute_uk_childcare_100k.py b/pipeline/compute_uk_childcare_100k.py index 10fd65e20..23841efe5 100644 --- a/pipeline/compute_uk_childcare_100k.py +++ b/pipeline/compute_uk_childcare_100k.py @@ -10,10 +10,13 @@ 2026), technical appendix Tables 2-3: the free hours and Tax-Free Childcare -This script runs the managed policyengine-uk simulation on the certified -populace-uk bundle (the default replay variant, data/uk/certified_bundle.json) -through the same preflight as the variant's event replays -(pipeline/compute_uk_event.py), and writes one artifact with weighted SUMS +This script runs the managed policyengine-uk simulation on a certified +pair: by default the certified populace-uk bundle (the default replay +variant, data/uk/certified_bundle.json), or with --variant another replay +variant or a certified pair in data/uk/certified_pairs.json +(pipeline/uk_certified_pairs.py), through the same preflight as a variant's +event replays (pipeline/compute_uk_event.py), and writes one artifact per +pair with weighted SUMS and COUNTS per world and year, UK-wide and for people in English households. Every ratio and delta is derived downstream (scorecard_db/ingest_uk_childcare_100k.py). @@ -37,9 +40,10 @@ how much of the static cost turns on the dataset's hours-usage draw (microcosm#1126); they are never attached to a claim. -Offline and pinned, through compute_uk_event's preflight for the variant -(the legacy one or a materialised one): the installed wheels checked -against the variant record before the engine is imported (versions, RECORD +Offline and pinned, through compute_uk_event's preflight for the record +(the legacy variant, or a materialised variant or certified pair): the +installed wheels checked against the record before the engine is imported +(versions, RECORD hashes, the packaged bundle manifest, Python), each engine package imported from its own distribution, Hugging Face offline mode, and the runtime release bundle equal to the record's. Each simulation must report as its @@ -49,7 +53,7 @@ variant's run in the history. PYTHONPATH=. .venv-pe289m/bin/python pipeline/compute_uk_childcare_100k.py - PYTHONPATH=. /bin/python pipeline/compute_uk_childcare_100k.py --variant + PYTHONPATH=. /bin/python pipeline/compute_uk_childcare_100k.py --variant """ from __future__ import annotations @@ -72,6 +76,7 @@ from pipeline import compute_uk_ab2025 as ab from pipeline import compute_uk_event as event from pipeline import compute_uk_obr_costings as fiscal +from pipeline import uk_certified_pairs as pairs from pipeline import uk_replay_variants as variants OUTPUT_DIR = ROOT / "results" / "uk" / "childcare_100k" @@ -86,6 +91,9 @@ "free_hours_and_tfc": (THIRTY_HOURS_LIMIT, TFC_LIMIT), } HOURS_USAGE = "maximum_extended_childcare_hours_usage" +# Each child's own weekly hours cap, beside the family's HOURS_USAGE (both +# pins take the smaller); its engine default is FULL_USE_HOURS. +CHILD_HOURS = "max_free_entitlement_hours_used" FULL_USE_HOURS = 30.0 # Within a data bundle the scorecard's comparisons view prefers these runs # (scorecard_db/db.py, run_id LIKE 'platform-grid-%', which ignores ASCII @@ -102,7 +110,7 @@ ) # Inputs whose presence in the dataset is recorded: holder periods filled at # construction, before any calculation, are the dataset's own values. -DATASET_INPUTS = (HOURS_USAGE, *TAKE_UP, "childcare_expenses") +DATASET_INPUTS = (HOURS_USAGE, CHILD_HOURS, *TAKE_UP, "childcare_expenses") # The three free early-years entitlements (policyengine-uk's gov_spending # lists all three) and Tax-Free Childcare. @@ -113,9 +121,10 @@ ) TFC = "tax_free_childcare" SUMMED = (*FREE_HOURS, TFC, "gov_spending", "gov_tax") -# Both offers are England's, but policyengine-uk 2.89.2 applies Tax-Free -# Childcare UK-wide (only the extended hours test country == ENGLAND), so -# the childcare heads are also summed for people in English households. +# Both offers are England's, but policyengine-uk applies Tax-Free Childcare +# UK-wide (it has no country test; the three free-hours offers test +# country == ENGLAND), so the childcare heads are also summed for people in +# English households. HEADS = (*FREE_HOURS, TFC) ENGLAND = "ENGLAND" # Variables whose engine metadata is recorded (the pe_meta contract): every @@ -128,7 +137,7 @@ "country", *TAKE_UP, "childcare_expenses", - "max_free_entitlement_hours_used", + CHILD_HOURS, "extended_childcare_entitlement_eligible", "extended_childcare_entitlement_meets_income_requirements", "universal_childcare_entitlement_eligible", @@ -235,6 +244,9 @@ def year_block(sim, year: int) -> dict: sums = {v: float(sim.calculate(v, year).sum()) for v in SUMMED} masks = england_masks(sim, year) ext, w_bu = _weighted(sim.calculate("extended_childcare_entitlement", year)) + if not w_bu.sum() > 0: + # every share and mean below divides by the benefit units' weight + raise SystemExit(f"{year}: benefit-unit weights sum to {w_bu.sum()}") hours, _ = _weighted(sim.calculate(HOURS_USAGE, year)) receiving = ext > 0 age, w_p = _weighted(sim.calculate("age", year)) @@ -406,7 +418,8 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--output-dir", type=Path, default=OUTPUT_DIR) ap.add_argument( "--variant", - help="a certified variant in data/uk/replay_variants.json (default: its default)", + help="a certified variant in data/uk/replay_variants.json (default: its " + "default) or a certified pair (pair-...) in data/uk/certified_pairs.json", ) ap.add_argument( "--dataset-file", @@ -417,7 +430,7 @@ def main(argv: list[str] | None = None) -> int: # absolute before a variant preflight moves into its runtime directory args.output_dir = args.output_dir.resolve() - variant = variants.get_variant(args.variant) + variant = pairs.get_record(args.variant) run_id = args.run_id or default_run_id(variant["variant_id"], started) if reserved_run_id(run_id): raise SystemExit( @@ -445,8 +458,9 @@ def main(argv: list[str] | None = None) -> int: } else: # The same checks for a variant whose dataset is materialised into - # its runtime directory (the event runner's variant preflight). - pre = event.variant_preflight( + # its runtime directory (the event runner's variant preflight), or + # for a certified pair, the pair checks and then the same path. + pre = pairs.preflight( variant, dataset_source=args.dataset_file.resolve() if args.dataset_file else None, ) diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index 43e251760..b23018e72 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -581,6 +581,15 @@ def variant_preflight(variant: dict, *, dataset_source: Path | None = None) -> d 6.x materialises its default dataset. Spawned year workers inherit both. """ installed = check_variant_environment(variant) + return materialized_preflight(variant, installed, dataset_source=dataset_source) + + +def materialized_preflight( + variant: dict, installed: dict, *, dataset_source: Path | None = None +) -> dict: + """variant_preflight after its environment check, for a record whose + installed wheels were already verified against it: a replay variant, or + a lane's certified pair (pipeline/uk_certified_pairs.py).""" variants.isolate_certification_environment() fiscal.configure_offline() worlds.configure_offline() diff --git a/pipeline/uk_certified_pairs.py b/pipeline/uk_certified_pairs.py new file mode 100644 index 000000000..3aa64631c --- /dev/null +++ b/pipeline/uk_certified_pairs.py @@ -0,0 +1,140 @@ +"""Certified pairs a lane runs on outside the replay variant index. + +A certified pair is a policyengine.py release's certified bundle: the +wrapper, the model package and the data build its packaged bundle manifest +certifies together. data/uk/certified_pairs.json records each one in the +replay variant's identity shape (pipeline/uk_replay_variants.py), under the +id ``pair-``, so a lane's computation can apply the replay +variant's checks to it before the release-replay pipeline qualifies the +pair (docs/uk_replay/RELEASE_REPLAY.md). + +No event replay runs on a certified pair: it is not in +data/uk/replay_variants.json, it has no result root or layout, and nothing +here writes either file. A variant the pipeline later qualifies for the same +pair has its own id and identity (its own requirements file), so the pair +record stays: it is the identity the lane's runs on it were checked +against. Experiment populations (docs/uk_replay/POLICY_EXPERIMENTS.md) stay +separate: their results are never ingested. + + from pipeline import uk_certified_pairs as pairs + record = pairs.get_record("pair-pe6.11.0_uk2.132.6_...") # or a variant id + pre = pairs.preflight(record, dataset_source=None) +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from pipeline import uk_replay_variants as variants + +ROOT = variants.ROOT +PAIRS_PATH = ROOT / "data" / "uk" / "certified_pairs.json" +KIND = "uk_certified_pairs" +STATUS = "certified_pair" +PREFIX = "pair-" +# What a pair record holds; a variant's status, layout and result root are +# not among them. +FIELDS = frozenset( + { + "certification", + "data_years", + "data_years_evidence", + "executor", + "identity", + "identity_sha256", + "note", + "pair_id", + "requirements", + "status", + } +) + + +def load_pairs(path: Path = PAIRS_PATH) -> dict: + document = json.loads(path.read_bytes()) + if document.get("schema_version") != 1 or document.get("kind") != KIND: + raise variants.VariantError( + f"{path.name}: schema_version must be 1 and kind {KIND}" + ) + pairs = document.get("pairs") + if not isinstance(pairs, list): + raise variants.VariantError(f"{path.name}: pairs must be a list") + ids = [pair.get("pair_id") for pair in pairs] + if len(ids) != len(set(ids)): + raise variants.VariantError(f"{path.name} repeats a pair_id") + return document + + +def validate_pair(record: dict, *, root: Path = ROOT) -> None: + """Engine-free: the record is one certified pairing (the replay variant's + identity checks) and a certified pair, not a replay variant.""" + pair_id = record.get("pair_id") + extra = sorted(set(record) - FIELDS) + missing = sorted(FIELDS - set(record)) + if extra or missing: + raise variants.VariantError( + f"{pair_id}: a certified pair holds exactly {sorted(FIELDS)}; " + f"extra {extra}, missing {missing}" + ) + if record["status"] != STATUS: + raise variants.VariantError(f"{pair_id}: status must be {STATUS}") + if not str(pair_id).startswith(PREFIX): + raise variants.VariantError(f"{pair_id}: a pair_id starts {PREFIX}") + variants.validate_identity(record, pair_id.removeprefix(PREFIX), root=root) + + +def view(record: dict) -> dict: + """The fields the variant checks and the lane read, under the pair's own + id. In memory only: never written, never added to the variant index.""" + return { + "variant_id": record["pair_id"], + "certified_pair": True, + "certification": record["certification"], + "data_years": record["data_years"], + "executor": record["executor"], + "identity": record["identity"], + "identity_sha256": record["identity_sha256"], + "requirements": record["requirements"], + "status": record["status"], + } + + +def is_pair(record: dict) -> bool: + return record.get("certified_pair") is True + + +def get_record( + record_id: str | None = None, + *, + variants_path: Path = variants.VARIANTS_PATH, + pairs_path: Path = PAIRS_PATH, + root: Path = ROOT, +) -> dict: + """A certified pair's view (an id starting pair-), else a certified + replay variant (None: the variant file's default), each validated.""" + if record_id is None or not record_id.startswith(PREFIX): + return variants.get_variant(record_id, path=variants_path, root=root) + for record in load_pairs(pairs_path)["pairs"]: + if record["pair_id"] == record_id: + validate_pair(record, root=root) + return view(record) + raise variants.VariantError(f"no certified pair {record_id!r}") + + +def preflight(record: dict, *, dataset_source: Path | None = None) -> dict: + """The event runner's preflight for a non-legacy record: a replay + variant's as it is (compute_uk_event.variant_preflight); for a certified + pair, the pair checks, then the same installed-environment verification + and materialised-dataset path.""" + from pipeline import compute_uk_event as event + + if not is_pair(record): + return event.variant_preflight(record, dataset_source=dataset_source) + if record["status"] != STATUS: + raise variants.VariantError(f"{record['variant_id']}: not a certified pair") + variants.validate_identity(record, record["variant_id"].removeprefix(PREFIX)) + installed = variants.verify_installed(record) + return event.materialized_preflight( + record, installed, dataset_source=dataset_source + ) diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py index b9bd5e8ad..49e3a7a68 100644 --- a/pipeline/uk_replay_variants.py +++ b/pipeline/uk_replay_variants.py @@ -232,12 +232,69 @@ def get_variant( def is_legacy(record: dict) -> bool: - """The populace-2023 world whose results keep their original layout.""" - return record["layout"] == "legacy_events" + """The populace-2023 world whose results keep their original layout. A + lane's certified pair (pipeline/uk_certified_pairs.py) has no layout.""" + return record.get("layout") == "legacy_events" def validate_record(record: dict, *, root: Path = ROOT) -> None: - """Engine-free consistency: the record is one certified pairing.""" + """Engine-free consistency: the record is one certified pairing, and a + replay variant (its status, layout and result root).""" + validate_identity(record, record.get("variant_id"), root=root) + identity = record["identity"] + bundle = identity["release_bundle"] + dataset = identity["dataset"] + uk = identity["packages"]["policyengine-uk"]["version"] + problems = [] + + def need(condition: bool, message: str) -> None: + if not condition: + problems.append(message) + + need(record.get("status") in STATUSES, "status") + need(record.get("layout") in LAYOUTS, "layout") + expected_root = ( + LEGACY_RESULT_ROOT + if record.get("layout") == "legacy_events" + else f"{REPLAY_RESULT_ROOT}/{record.get('variant_id')}" + ) + need( + record.get("result_root") == expected_root, + f"result_root must be {expected_root}", + ) + if record.get("layout") == "legacy_events": + legacy = json.loads( + (root / "data" / "uk" / "certified_bundle.json").read_bytes() + ) + need( + legacy.get("repo_id") == dataset["repo_id"] + and legacy.get("artifact") == dataset["path"] + and legacy.get("revision") == dataset["revision"] + and legacy.get("revision") == bundle.get("certified_data_build_id") + and legacy.get("sha256") == dataset["sha256"] + and legacy.get("size_bytes") == dataset["size_bytes"], + "the legacy variant differs from data/uk/certified_bundle.json", + ) + need( + any( + row.get("name") == "policyengine-uk" + and row.get("specifier") == f"=={uk}" + for row in legacy.get("compatible_model_packages", []) + ), + "data/uk/certified_bundle.json does not declare this model version", + ) + if problems: + raise VariantError(f"{record.get('variant_id')}: " + "; ".join(problems)) + + +def validate_identity( + record: dict, record_id: str | None, *, root: Path = ROOT +) -> None: + """Engine-free consistency of one certified pairing, whatever indexes it + (a replay variant here, or a lane's certified pair in + pipeline/uk_certified_pairs.py): the identity and its digest, the id it + implies, the certification, the dataset, the executors, the data years + and the pinned requirements.""" problems = [] def need(condition: bool, message: str) -> None: @@ -251,7 +308,7 @@ def need(condition: bool, message: str) -> None: dataset = identity["dataset"] certification = record["certification"] except (KeyError, TypeError) as exc: - raise VariantError(f"variant record is missing {exc}") from exc + raise VariantError(f"{record_id}: record is missing {exc}") from exc if not all( isinstance(part, dict) for part in (identity, packages, bundle, dataset, certification) @@ -285,12 +342,12 @@ def need(condition: bool, message: str) -> None: uk = packages["policyengine-uk"]["version"] wrapper = packages["policyengine"]["version"] need( - record.get("variant_id") + record_id in ( variant_slug(identity), f"{variant_slug(identity)}--{identity_sha256(identity)[:10]}", ), - "variant_id differs from its identity", + "the variant_id or pair_id differs from its identity", ) need( record.get("identity_sha256") == identity_sha256(identity), @@ -354,8 +411,6 @@ def need(condition: bool, message: str) -> None: and bool(re.fullmatch(r"3\.[0-9]+", identity.get("python", ""))), "identity.python", ) - need(record.get("status") in STATUSES, "status") - need(record.get("layout") in LAYOUTS, "layout") need( isinstance(record.get("executor"), list) and bool(record["executor"]) @@ -370,15 +425,6 @@ def need(condition: bool, message: str) -> None: and years == list(range(years[0], years[-1] + 1)), "data_years must be consecutive calendar years", ) - expected_root = ( - LEGACY_RESULT_ROOT - if record.get("layout") == "legacy_events" - else f"{REPLAY_RESULT_ROOT}/{record.get('variant_id')}" - ) - need( - record.get("result_root") == expected_root, - f"result_root must be {expected_root}", - ) requirements = record.get("requirements") or {} need( identity.get("requirements_sha256") == requirements.get("sha256"), @@ -403,29 +449,8 @@ def need(condition: bool, message: str) -> None: pins.get(name) == packages[name]["version"], f"requirements pin {name}", ) - if record.get("layout") == "legacy_events": - legacy = json.loads( - (root / "data" / "uk" / "certified_bundle.json").read_bytes() - ) - need( - legacy.get("repo_id") == dataset["repo_id"] - and legacy.get("artifact") == dataset["path"] - and legacy.get("revision") == dataset["revision"] - and legacy.get("revision") == bundle.get("certified_data_build_id") - and legacy.get("sha256") == dataset["sha256"] - and legacy.get("size_bytes") == dataset["size_bytes"], - "the legacy variant differs from data/uk/certified_bundle.json", - ) - need( - any( - row.get("name") == "policyengine-uk" - and row.get("specifier") == f"=={uk}" - for row in legacy.get("compatible_model_packages", []) - ), - "data/uk/certified_bundle.json does not declare this model version", - ) if problems: - raise VariantError(f"{record.get('variant_id')}: " + "; ".join(problems)) + raise VariantError(f"{record_id}: " + "; ".join(problems)) # --- the installed environment --------------------------------------------- diff --git a/results/uk/childcare_100k/construction_annotations.json b/results/uk/childcare_100k/construction_annotations.json index 022caaed4..7c702abf8 100644 --- a/results/uk/childcare_100k/construction_annotations.json +++ b/results/uk/childcare_100k/construction_annotations.json @@ -32,7 +32,8 @@ "static_because": "policyengine-uk 2.89.2's labour-supply elasticities are zero, so employment_income_behavioral_response is zero", "hours_draw": "microcosm#1126 traces this draw and puts DfE's mean at about 28.5 hours.", "diagnosis": "microcosm#1126 identifies this draw as the main reason PolicyEngine's free-hours spending is low and derives DfE's mean as about 28.5 hours. The fix merged in microcosm on 16 October 2026 (microcosm#1299); this certified pair was built on 19 June 2026, before it. policyengine.py#685 (merged 21 October 2026) certified a Microcosm pair with the fixed draw, national cut 6cba5d71 with policyengine-uk 2.132.6, which this lane has not run yet (policyengine-scorecard#228). PolicyEngine's published dashboard figure, on policyengine.org from 12 to 21 October 2026, ran on the 2 October Microcosm release that microcosm#1126 examines, with the same draw. Two smaller PolicyEngine-side factors are recorded on the result: the universal-hours switch-off (policyengine-uk#1930, changed after the pin by policyengine-uk#2177) and no hours for children under 1.", - "basis": "every note whose variants name this pair, read against policyengine_uk 2.89.2 at the pin and results/uk/childcare_100k/.json; static_because: the centax-static-vs-static basis; hours_draw and diagnosis: https://github.com/PolicyEngine/microcosm/issues/1126, https://github.com/PolicyEngine/microcosm/pull/1299 (merged 2026-10-16), https://github.com/PolicyEngine/policyengine.py/pull/685 (merged 2026-10-21; policyengine.py 6.11.0, released 2026-10-25, still pins that pair in src/policyengine/data/bundle/uk.trace.tro.jsonld), sources/centax-childcare-100k/pe/published_dashboard.json (live), this variant's dataset revision in data/uk/replay_variants.json (built 2026-06-19T02:37:11Z), and the pe-universal-switch-off and pe-under-ones notes" + "basis": "every note whose variants name this pair, read against policyengine_uk 2.89.2 at the pin and results/uk/childcare_100k/.json; static_because: the centax-static-vs-static basis; hours_draw and diagnosis: https://github.com/PolicyEngine/microcosm/issues/1126, https://github.com/PolicyEngine/microcosm/pull/1299 (merged 2026-10-16), https://github.com/PolicyEngine/policyengine.py/pull/685 (merged 2026-10-21; policyengine.py 6.11.0, released 2026-10-25, still pins that pair in src/policyengine/data/bundle/uk.trace.tro.jsonld), sources/centax-childcare-100k/pe/published_dashboard.json (live), this variant's dataset revision in data/uk/replay_variants.json (built 2026-06-19T02:37:11Z), and the pe-universal-switch-off and pe-under-ones notes; child_hours_input: the person, benunit and household table columns of populace_uk_2023.h5 (sha256 f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833), read from the file's HDF metadata on 2026-10-26, and policyengine_uk 2.89.2 variables/gov/dfe/max_free_entitlement_hours_used.py (default_value 30) and extended_childcare_entitlement/extended_childcare_entitlement.py (min_ of both caps)", + "child_hours_input": "max_free_entitlement_hours_used, each child's own weekly hours cap beside the family's maximum_extended_childcare_hours_usage, is not a column of this pair's dataset, so every child is at the engine default of 30 hours in every world. This run predates the artifact's record of that input (dataset_inputs)." } } } diff --git a/scorecard_db/ingest_uk_childcare_100k.py b/scorecard_db/ingest_uk_childcare_100k.py index 21a3e6750..f3ebe97b5 100644 --- a/scorecard_db/ingest_uk_childcare_100k.py +++ b/scorecard_db/ingest_uk_childcare_100k.py @@ -16,7 +16,9 @@ Inputs: the adapters' tidy rows (data/externals/.json, QC'd there), each source's annotations.json, every certified-pair artifact results/uk/childcare_100k/.json (pipeline/compute_uk_childcare_100k.py), -each checked against its own record in data/uk/replay_variants.json, with +each checked against its own record in data/uk/replay_variants.json or, for +a certified pair the release-replay pipeline has not qualified, +data/uk/certified_pairs.json (pipeline/uk_certified_pairs.py), with their construction_annotations.json, and the published dashboard figure in sources/centax-childcare-100k/pe/published_dashboard.json. @@ -101,6 +103,7 @@ CBP_DIR = REPO / "sources" / "cbp-rates-and-wrongs" RESULTS_DIR = REPO / "results" / "uk" / "childcare_100k" VARIANTS = REPO / "data" / "uk" / "replay_variants.json" +PAIRS = REPO / "data" / "uk" / "certified_pairs.json" CERTIFIED = REPO / "data" / "uk" / "certified_bundle.json" CONSTRUCTION = RESULTS_DIR / "construction_annotations.json" # The artifact schema this ingest reads (pipeline/compute_uk_childcare_100k.py). @@ -132,6 +135,9 @@ TFC = "tax_free_childcare" VARIABLES = {CENTAX: list(FREE_HOURS), CBP: [*FREE_HOURS, TFC]} HOURS_USAGE_INPUT = "maximum_extended_childcare_hours_usage" +# Each child's own weekly hours cap beside the family's (the engine takes the +# smaller); full use needs it at its default, FULL_USE_HOURS. +CHILD_HOURS_INPUT = "max_free_entitlement_hours_used" # the extended offer's weekly hours: the full-use diagnostic sets every # family's usage to them FULL_USE_HOURS = 30.0 @@ -430,7 +436,7 @@ def check_artifact(art: dict, path: Path, variant: dict) -> None: wrong = sorted(k for k in want if got[k] != want[k]) if wrong: raise ValueError(f"{path.name}: not {vid}'s certified pairing: {wrong}") - if variant["layout"] == "legacy_events": + if variant.get("layout") == "legacy_events": cert = json.loads(CERTIFIED.read_text()) if (dataset["revision"], dataset["sha256"]) != ( cert["revision"], @@ -454,23 +460,41 @@ def check_artifact(art: dict, path: Path, variant: dict) -> None: raise ValueError(f"{path.name}: artifact is not a static run") +def run_records(variants_path: Path = VARIANTS, pairs_path: Path = PAIRS) -> dict: + """id -> record, for every record a run of this lane may name: each + certified replay variant, and each certified pair under its pair- id + (pipeline/uk_certified_pairs.py validates both files; a test checks the + committed ones).""" + index = json.loads(variants_path.read_text()) + records = { + v["variant_id"]: v for v in index["variants"] if v["status"] == "certified" + } + for pair in json.loads(pairs_path.read_text())["pairs"]: + if pair.get("status") == "certified_pair": + records[pair["pair_id"]] = {**pair, "variant_id": pair["pair_id"]} + return records + + def load_artifacts( - results_dir: Path = RESULTS_DIR, variants_path: Path = VARIANTS + results_dir: Path = RESULTS_DIR, + variants_path: Path = VARIANTS, + pairs_path: Path = PAIRS, ) -> list[tuple[dict, Path]]: - """Every certified-pair artifact, each checked against its own certified - variant record, oldest first. Each claim's latest result is the newest - run that computed its year, so run ids and UTC computed_at times must be - distinct.""" - index = json.loads(variants_path.read_text()) - records = {v["variant_id"]: v for v in index["variants"]} + """Every certified-pair artifact, each checked against its own record (a + certified variant or a certified pair), oldest first. Each claim's latest + result is the newest run that computed its year, so run ids and UTC + computed_at times must be distinct.""" + records = run_records(variants_path, pairs_path) out = [] for path in sorted(results_dir.glob("*.json")): if path.name == CONSTRUCTION.name: continue art = json.loads(path.read_text()) variant = records.get(art.get("variant_id")) - if variant is None or variant["status"] != "certified": - raise ValueError(f"{path.name}: names no certified variant") + if variant is None: + raise ValueError( + f"{path.name}: names no certified variant or certified pair" + ) check_artifact(art, path, variant) out.append((art, path)) out.sort(key=lambda a: a[0]["computed_at"]) @@ -569,7 +593,8 @@ def _pe_missing( ) return ( f"fiscal year {fy} is outside the data years of every certified pair this " - f"lane has run ({spans}; data/uk/replay_variants.json)" + f"lane has run ({spans}; data/uk/replay_variants.json, " + "data/uk/certified_pairs.json)" ) @@ -577,12 +602,13 @@ def stage( notes: dict | None = None, results_dir: Path = RESULTS_DIR, variants_path: Path = VARIANTS, + pairs_path: Path = PAIRS, ) -> tuple[list[ExternalScore], dict, list[tuple[dict, Path]]]: """(claims, drop tally, every certified artifact oldest first). Each artifact's pair must be a verified one; no claim id depends on which runs exist.""" notes = notes or load_notes() - artifacts = load_artifacts(results_dir, variants_path) + artifacts = load_artifacts(results_dir, variants_path, pairs_path) for art, _ in artifacts: verified_pair(notes, art) scores: list[ExternalScore] = [] @@ -856,10 +882,22 @@ def measured_notes(art: dict, source: str, year: int, fy: str, pair: dict) -> li ) + f"; {HOURS_USAGE_INPUT} is " + ( - "the dataset's own input." + "the dataset's own input" if supplied.get(HOURS_USAGE_INPUT) - else "the engine's default (not in the dataset for this year)." + else "the engine's default (not in the dataset for this year)" + ) + + ( + "" + if CHILD_HOURS_INPUT not in supplied + else f"; {CHILD_HOURS_INPUT}, each child's own cap, is " + + ( + "the dataset's own input" + if supplied[CHILD_HOURS_INPUT] + else f"the engine's default of {FULL_USE_HOURS:g} (not in the " + "dataset for this year)" + ) ) + + "." ), "Executed limits on 30 April {y}: 30 hours {a:g}, Tax-Free Childcare {t:g} (1e+100 is the " "repo's no-limit sentinel, compute_uk_ab2025.NO_LIMIT).".format( @@ -870,7 +908,7 @@ def measured_notes(art: dict, source: str, year: int, fy: str, pair: dict) -> li t=w["limits_executed"]["gov.hmrc.tax_free_childcare.income.income_limit"], ), ] - full = full_use_cost(art, year) if source == CENTAX else None + full = full_use_cost(art, year, pair) if source == CENTAX else None if full is not None: notes.append( f"Diagnostic, not attached: with every family at the full {FULL_USE_HOURS:g} hours (the " @@ -927,12 +965,16 @@ def _same_reform(reform, main, year: int) -> bool: ) -def full_use_cost(art: dict, year: int) -> float | None: +def full_use_cost(art: dict, year: int, pair: dict) -> float | None: """The reform's free-hours cost with every family at the full 30 hours, from the run's two full-use worlds, or None when it ran none for this year. Each world must record what the text states: - the hours input set to 30 in this year, with every holder that reads it (FULL_USE_DEPENDENTS) still empty, and read back at 30; + - each child's own hours cap (CHILD_HOURS_INPUT) at its engine default + of 30: the world records the dataset supplying it in no year, or, for + a run that predates that record, the pair's verified_pairs entry + states it from the dataset (child_hours_input); - the main world's reform: the same limits removed on 30 April of the year, under the repo's NO_LIMIT sentinel; - the main world's executed limits, read back from the engine on that @@ -957,8 +999,12 @@ def full_use_cost(art: dict, year: int) -> float | None: executed = block.get("limits_executed") removed = set(reform or {}) receiving = hours.get("mean_benunits_receiving_extended") + inputs = d.get("dataset_inputs") or {} evidence = { "variable": o.get("variable") == HOURS_USAGE_INPUT, + "child hours": inputs[CHILD_HOURS_INPUT] == [] + if CHILD_HOURS_INPUT in inputs + else bool(pair.get("child_hours_input")), "value": o.get("value") == FULL_USE_HOURS, "year set": year in o.get("set_for_years", []), "set first": isinstance(deps, dict) @@ -1174,7 +1220,7 @@ def diagnoses( if hours is not None else _no_recipients(c["fy"]) ) - full = full_use_cost(art, year) + full = full_use_cost(art, year, pair) if full is None: raise ValueError( f"{art['variant_id']}: the diagnosis needs the full-use diagnostic for " diff --git a/tests/test_uk_certified_pairs.py b/tests/test_uk_certified_pairs.py new file mode 100644 index 000000000..25da4ed94 --- /dev/null +++ b/tests/test_uk_certified_pairs.py @@ -0,0 +1,212 @@ +"""Certified pairs: lane records in the replay variant's identity shape. + +Engine-free: the committed registry is validated as stored, mutations are +made on copies, and the preflight's engine steps are stubbed. + +Invariants: +- every committed pair validates, under an id pair-, and + no pair is a replay variant (neither file names the other's ids); +- a pair binds its identity: changing any one value in it, without + resealing the digest, is refused (property test); +- a pair's checks are the replay variant's identity checks: given a + variant's status, layout and result root, the same record passes + validate_record (differential), and ingest reads the same records; +- the preflight routes a pair through the pair checks and the materialised + path, and a variant through the event runner's own preflight, never the + other way round. +""" + +import copy +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import compute_uk_event as event +from pipeline import uk_certified_pairs as pairs +from pipeline import uk_replay_variants as rv +from scorecard_db import ingest_uk_childcare_100k as lane + +PAIR = "pair-pe6.11.0_uk2.132.6_microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + + +def stored(pair_id=PAIR) -> dict: + return copy.deepcopy( + next(p for p in pairs.load_pairs()["pairs"] if p["pair_id"] == pair_id) + ) + + +def test_committed_pairs_validate_under_their_slug_and_are_no_variants(): + document = pairs.load_pairs() + assert document["kind"] == pairs.KIND + ids = [p["pair_id"] for p in document["pairs"]] + assert PAIR in ids + variant_ids = {v["variant_id"] for v in rv.load_document()["variants"]} + for record in document["pairs"]: + pairs.validate_pair(record) + assert record["pair_id"] == pairs.PREFIX + rv.variant_slug(record["identity"]) + assert record["pair_id"] not in variant_ids + assert record["identity_sha256"] == rv.identity_sha256(record["identity"]) + assert not any(v.startswith(pairs.PREFIX) for v in variant_ids) + + +def test_get_record_resolves_a_pair_a_variant_and_the_default(): + pair = pairs.get_record(PAIR) + assert pairs.is_pair(pair) and pair["variant_id"] == PAIR + assert pair["identity"] == stored()["identity"] + assert "layout" not in pair and "result_root" not in pair + assert not rv.is_legacy(pair) + default = rv.load_document()["default_variant"] + assert pairs.get_record(None) == rv.get_variant(default) + assert pairs.get_record(default) == rv.get_variant(default) + assert not pairs.is_pair(pairs.get_record(default)) + with pytest.raises(rv.VariantError, match="no certified pair"): + pairs.get_record(PAIR + "x") + with pytest.raises(rv.VariantError, match="no replay variant"): + pairs.get_record(PAIR.removeprefix(pairs.PREFIX)) + + +@pytest.mark.parametrize( + "change, message", + [ + (lambda r: r.update(layout="replay_namespace"), "holds exactly"), + (lambda r: r.update(result_root="results/uk/replays/x"), "holds exactly"), + (lambda r: r.update(variant_id=r["pair_id"]), "holds exactly"), + (lambda r: r.pop("data_years_evidence"), "holds exactly"), + (lambda r: r.update(status="certified"), "status must be"), + ( + lambda r: r.update(pair_id=r["pair_id"].removeprefix(pairs.PREFIX)), + "starts pair-", + ), + (lambda r: r.update(pair_id=r["pair_id"] + "x"), "differs from its identity"), + (lambda r: r.update(identity_sha256="0" * 64), "identity_sha256 differs"), + (lambda r: r["requirements"].update(sha256="0" * 64), "requirements"), + (lambda r: r.update(executor=["cloud"]), "executor"), + (lambda r: r.update(data_years=[2024, 2026]), "consecutive"), + ( + lambda r: r["certification"].update(certified_for_model_version="2.89.2"), + "does not name this model version", + ), + ], +) +def test_a_pair_is_refused_for_each_broken_field(change, message): + record = stored() + change(record) + with pytest.raises(rv.VariantError, match=message): + pairs.validate_pair(record) + + +def _leaves(value, path=()): + """(path, value) for every string or integer in a nested record.""" + if isinstance(value, dict): + for key in sorted(value): + yield from _leaves(value[key], (*path, key)) + elif isinstance(value, str) or ( + isinstance(value, int) and not isinstance(value, bool) + ): + yield path, value + + +_IDENTITY_LEAVES = list(_leaves(stored()["identity"])) + + +@settings(max_examples=60, deadline=None) +@given( + leaf=st.sampled_from(_IDENTITY_LEAVES), + suffix=st.text(alphabet="0123456789abcdefxyz.-", min_size=1, max_size=4), +) +def test_any_change_to_a_pairs_identity_is_refused(leaf, suffix): + """The digest binds the identity: one changed value, with the stored + identity_sha256 kept, never validates.""" + path, value = leaf + record = stored() + node = record["identity"] + for key in path[:-1]: + node = node[key] + node[path[-1]] = value + 1 if isinstance(value, int) else value + suffix + with pytest.raises(rv.VariantError): + pairs.validate_pair(record) + + +def test_a_pairs_checks_are_the_replay_variants_identity_checks(): + """Differential: the stored pair, given a replay variant's own fields, + is a valid variant record, so validate_identity is the whole of what the + variant checks apply to it.""" + record = stored() + variant_id = record.pop("pair_id").removeprefix(pairs.PREFIX) + record.update( + variant_id=variant_id, + status="certified", + layout="replay_namespace", + result_root=f"{rv.REPLAY_RESULT_ROOT}/{variant_id}", + ) + rv.validate_record(record) + + +def test_ingest_reads_the_records_the_pipeline_resolves(): + """Differential: scorecard_db reads both files itself (it does not + import pipeline/); for each id they must give the same record.""" + read = lane.run_records() + certified = [ + v["variant_id"] + for v in rv.load_document()["variants"] + if v["status"] == "certified" + ] + ids = [p["pair_id"] for p in pairs.load_pairs()["pairs"]] + assert sorted(read) == sorted([*certified, *ids]) + for record_id in read: + resolved = pairs.get_record(record_id) + for field in ("identity", "identity_sha256", "data_years", "variant_id"): + assert read[record_id][field] == resolved[field], (record_id, field) + + +def test_the_preflight_routes_each_record_its_own_way(monkeypatch): + calls = [] + monkeypatch.setattr( + event, + "variant_preflight", + lambda record, dataset_source=None: calls.append(("variant", record)), + ) + monkeypatch.setattr( + event, + "materialized_preflight", + lambda record, installed, dataset_source=None: calls.append( + ("materialized", record, installed) + ), + ) + monkeypatch.setattr(rv, "verify_installed", lambda record: {"verified": True}) + variant = rv.get_variant(rv.load_document()["variants"][-1]["variant_id"]) + pairs.preflight(variant) + pair = pairs.get_record(PAIR) + pairs.preflight(pair, dataset_source=Path("x.h5")) + assert calls == [ + ("variant", variant), + ("materialized", pair, {"verified": True}), + ] + + +def test_the_preflight_refuses_a_pair_view_changed_after_resolution(monkeypatch): + monkeypatch.setattr( + rv, "verify_installed", lambda record: pytest.fail("checked a changed pair") + ) + pair = pairs.get_record(PAIR) + pair["identity"]["packages"]["policyengine-uk"]["version"] = "2.143.0" + with pytest.raises(rv.VariantError): + pairs.preflight(pair) + pair = pairs.get_record(PAIR) + pair["status"] = "certified" + with pytest.raises(rv.VariantError, match="not a certified pair"): + pairs.preflight(pair) + + +def test_the_registry_refuses_a_repeated_id_and_another_kind(tmp_path): + document = pairs.load_pairs() + path = tmp_path / "pairs.json" + path.write_text(json.dumps({**document, "pairs": document["pairs"] * 2})) + with pytest.raises(rv.VariantError, match="repeats a pair_id"): + pairs.load_pairs(path) + path.write_text(json.dumps({**document, "kind": "uk_replay_variants"})) + with pytest.raises(rv.VariantError, match="kind"): + pairs.load_pairs(path) diff --git a/tests/test_uk_childcare_100k.py b/tests/test_uk_childcare_100k.py index 4c139a10d..2eeca8c06 100644 --- a/tests/test_uk_childcare_100k.py +++ b/tests/test_uk_childcare_100k.py @@ -83,6 +83,24 @@ def artifact(): return lane.load_artifact()[0] +def _pair(art: dict) -> dict: + """The verified_pairs entry of the artifact's pair.""" + return lane.verified_pair(lane.load_notes(), art) + + +# every committed certified-pair artifact, oldest first +_ARTIFACTS = lane.load_artifacts() +_ARTIFACT_IDS = [a["variant_id"] for a, _ in _ARTIFACTS] +# the 19 June pair's run, which predates the artifact's record of each +# child's own cap on funded hours +JUNE = "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" + + +@pytest.fixture +def june(): + return next(a for a, _ in _ARTIFACTS if a["variant_id"] == JUNE) + + # --- the vendored sources --------------------------------------------------- @@ -366,17 +384,26 @@ def test_free_hours_moving_outside_england_is_refused(artifact): sums["extended_childcare_entitlement"] += 5e6 sums["gov_spending"] += 5e6 with pytest.raises(ValueError, match="outside England"): - lane.measured_notes(bad, lane.CENTAX, int(year), "fy", {}) + lane.measured_notes(bad, lane.CENTAX, int(year), "fy", _pair(bad)) -def test_artifact_records_the_inputs_and_metadata_it_read(artifact): +@pytest.mark.parametrize("pair", _ARTIFACTS, ids=_ARTIFACT_IDS) +def test_artifact_records_the_inputs_and_metadata_it_read(pair): + """Every committed run records the compute script's inputs, but a run + from before the child hours input was recorded lacks only that one, and + its pair's verified_pairs entry then states it from the dataset.""" + artifact, _ = pair compute = _module( "compute_uk_childcare_100k", REPO / "pipeline" / "compute_uk_childcare_100k.py" ) assert set(artifact["variable_metadata"]) == set(compute.METADATA) for group in ("worlds", "diagnostics"): for world in artifact[group].values(): - assert set(world["dataset_inputs"]) == set(compute.DATASET_INPUTS) + recorded = set(world["dataset_inputs"]) + if compute.CHILD_HOURS not in recorded: + assert _pair(artifact)["child_hours_input"] + recorded.add(compute.CHILD_HOURS) + assert recorded == set(compute.DATASET_INPUTS) for block in world["years"].values(): assert set(block["take_up"]) == set(compute.TAKE_UP) assert all(0 <= x <= 1 for x in block["take_up"].values()) @@ -443,6 +470,7 @@ class Unvouched(AssertionError): ("pipeline", "compute_uk_event", "event"), ("pipeline", "compute_uk_obr_costings", "fiscal"), ("pipeline", "modal_uk_event", None), + ("pipeline", "uk_certified_pairs", "pairs"), ("pipeline", "uk_replay_variants", "variants"), } ) @@ -505,7 +533,7 @@ class Unvouched(AssertionError): "get", "get_holder", "get_known_periods", - "get_variant", + "get_record", "getsourcefile", "insert", "is_legacy", @@ -546,7 +574,6 @@ class Unvouched(AssertionError): "values", "variables", "variant", - "variant_preflight", "verify_import_origins", "verify_runtime_dataset_sha256", "weights", @@ -1010,10 +1037,10 @@ def build(reform): return _StandIn(run, reform, years, source) monkeypatch.setattr(compute, "ROOT", out) - monkeypatch.setattr(compute.variants, "get_variant", _stub(variant)) + monkeypatch.setattr(compute.pairs, "get_record", _stub(variant)) monkeypatch.setattr(compute.variants, "is_legacy", _stub(False)) monkeypatch.setattr(compute.variants, "verify_import_origins", _stub(None)) - monkeypatch.setattr(compute.event, "variant_preflight", _stub(pre)) + monkeypatch.setattr(compute.pairs, "preflight", _stub(pre)) monkeypatch.setattr(compute.fiscal, "data_bundle_id", _stub("stand-in")) monkeypatch.setattr(compute.fiscal, "verify_runtime_dataset_sha256", _stub(sha)) monkeypatch.setattr(compute.fiscal, "assert_managed_bundle", _stub(None)) @@ -1805,6 +1832,8 @@ def _on_pair(art, variant, *, run_id, with_2031=False): dataset_sha256_after=sha, dataset_years=list(variant["data_years"]), ) + # as a run since the record began: no dataset values for it + world["dataset_inputs"].setdefault(lane.CHILD_HOURS_INPUT, []) if with_2031 and group == "worlds": world["years"]["2031"] = copy.deepcopy(world["years"]["2030"]) return out @@ -2045,6 +2074,10 @@ def _receiving_none(a): ].update({_LIMIT_30H: 100_000}), "no executed limits": _both(lambda d: d["years"]["2027"].pop("limits_executed")), "no receiving mean with recipients": _receiving_none, + # each child's own cap must be at its default of 30 too + "child hours from the dataset": _both( + lambda d: d["dataset_inputs"].update({lane.CHILD_HOURS_INPUT: ["2027"]}) + ), } @@ -2058,8 +2091,8 @@ def test_the_full_use_cost_rests_on_the_recorded_override(artifact): - d[_BASE_FULL]["years"][str(year)]["sums_gbp"][v] for v in lane.FREE_HOURS ) - assert lane.full_use_cost(artifact, year) == want > 0 - assert lane.full_use_cost(artifact, 2026) is None + assert lane.full_use_cost(artifact, year, _pair(artifact)) == want > 0 + assert lane.full_use_cost(artifact, 2026, _pair(artifact)) is None @pytest.mark.parametrize("what", sorted(_NOT_FULL_USE)) @@ -2071,9 +2104,9 @@ def test_no_text_claims_full_use_the_record_does_not_show(artifact, what): bad = copy.deepcopy(artifact) _NOT_FULL_USE[what](bad) with pytest.raises(ValueError, match="full use|beside the other"): - lane.full_use_cost(bad, 2027) + lane.full_use_cost(bad, 2027, _pair(bad)) with pytest.raises(ValueError, match="full use|beside the other"): - lane.measured_notes(bad, lane.CENTAX, 2027, "fy", {}) + lane.measured_notes(bad, lane.CENTAX, 2027, "fy", _pair(bad)) scores, _, artifacts = lane.stage() with pytest.raises(ValueError, match="full use|beside the other"): lane.diagnoses(scores, [(bad, artifacts[-1][1])]) @@ -2107,9 +2140,71 @@ def _per_year_windows(a): } ), "no benefit unit receiving": _zero_recipients, + "child hours recorded as the default": _both( + lambda d: d["dataset_inputs"].update({lane.CHILD_HOURS_INPUT: []}) + ), } +CHILD_KEY = lane.CHILD_HOURS_INPUT + + +def test_full_use_needs_the_child_hours_shown_by_the_run_or_the_pair(june): + """A run that predates the record of the child hours input shows full + use only through its pair's verified statement; a run that records the + input needs no such statement.""" + pair = _pair(june) + assert CHILD_KEY not in june["diagnostics"][_BASE_FULL]["dataset_inputs"] + silent = {k: v for k, v in pair.items() if k != "child_hours_input"} + with pytest.raises(ValueError, match="child hours"): + lane.full_use_cost(june, 2027, silent) + recorded = copy.deepcopy(june) + _both(lambda d: d["dataset_inputs"].update({CHILD_KEY: []}))(recorded) + assert lane.full_use_cost(recorded, 2027, silent) == lane.full_use_cost( + june, 2027, pair + ) + + +@pytest.mark.parametrize("periods, says", [([], "default of 30"), (["2027"], "own")]) +def test_the_notes_state_the_child_hours_input_a_run_recorded(june, periods, says): + """The take-up and usage note names the child hours input only when + the run recorded it, and says where its value came from.""" + old = lane.measured_notes(june, lane.CBP, 2027, "fy", _pair(june)) + assert not any(CHILD_KEY in n for n in old) + recorded = copy.deepcopy(june) + recorded["worlds"]["baseline"]["dataset_inputs"][CHILD_KEY] = periods + notes = lane.measured_notes(recorded, lane.CBP, 2027, "fy", _pair(recorded)) + (line,) = [n for n in notes if CHILD_KEY in n] + assert f"{CHILD_KEY}, each child's own cap, is the" in line + assert says in line and line.endswith(".") + assert [n for n in notes if CHILD_KEY not in n] == [ + n for n in old if "Take-up and usage" not in n + ] + + +class _Weightless(_StandIn): + """A stand-in whose benefit units weigh nothing in total.""" + + def calculate(self, variable, period): + series = super().calculate(variable, period) + if variable == "extended_childcare_entitlement": + series.weights = np.zeros_like(series.values, dtype=float) + return series + + +def test_a_population_with_no_benefit_unit_weight_is_refused(): + """The round-8 review's boundary: every share and mean year_block + writes divides by the benefit units' total weight, so a world where it + is zero stops the run instead of writing nan.""" + compute = _module( + "compute_uk_childcare_100k_weightless", + REPO / "pipeline" / "compute_uk_childcare_100k.py", + ) + sim = _Weightless(SimpleNamespace(calls=[], hours=[]), None, [2027], Path("w.h5")) + with pytest.raises(SystemExit, match="benefit-unit weights sum to 0"): + compute.year_block(sim, 2027) + + @pytest.mark.parametrize("what", sorted(_TRUE_FULL_USE)) def test_the_full_use_cost_keeps_every_true_record_shape(artifact, what): """The check refuses only what the record does not show: a reform @@ -2118,7 +2213,9 @@ def test_the_full_use_cost_keeps_every_true_record_shape(artifact, what): the offer (year_block then writes no receiving mean) all show full use.""" good = copy.deepcopy(artifact) _TRUE_FULL_USE[what](good) - assert lane.full_use_cost(good, 2027) == lane.full_use_cost(artifact, 2027) + assert lane.full_use_cost(good, 2027, _pair(good)) == lane.full_use_cost( + artifact, 2027, _pair(artifact) + ) def _current_law(a, year=2027): @@ -2150,7 +2247,7 @@ def test_no_family_receiving_under_current_law_is_stated_not_formatted(artifact) _current_law(none)[key] = made[key] assert lane.receiving_hours(none, 2027) is None stated = "no family receives the extended hours under current law in 2027-28" - notes = lane.measured_notes(none, lane.CENTAX, 2027, "2027-28", {}) + notes = lane.measured_notes(none, lane.CENTAX, 2027, "2027-28", _pair(none)) assert sum(stated in n for n in notes) == 1 assert not any("weekly hours under current law" in n for n in notes) scores, _, artifacts = lane.stage() @@ -2190,7 +2287,7 @@ def test_receiving_records_with_no_receiving_weight_are_stated_not_nan(artifact) for key in ("hours_usage", "counts", "records"): _current_law(none)[key] = made[key] assert lane.receiving_hours(none, 2027) is None - notes = lane.measured_notes(none, lane.CENTAX, 2027, "2027-28", {}) + notes = lane.measured_notes(none, lane.CENTAX, 2027, "2027-28", _pair(none)) stated = "no family receives the extended hours under current law in 2027-28" assert sum(stated in n for n in notes) == 1 assert not any("nan" in n for n in notes) @@ -2228,7 +2325,7 @@ def test_a_receiving_mean_must_agree_with_the_receiving_weight(artifact, mean, w with pytest.raises(ValueError, match="receiving-hours mean"): lane.receiving_hours(bad, 2027) with pytest.raises(ValueError, match="receiving-hours mean"): - lane.measured_notes(bad, lane.CENTAX, 2027, "fy", {}) + lane.measured_notes(bad, lane.CENTAX, 2027, "fy", _pair(bad)) def test_the_full_use_check_reads_the_compute_scripts_own_constants( From 44174a17965d221897b735878688006fe2e483d9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 06:22:01 -0700 Subject: [PATCH 2/5] Document certified pairs alongside replay variants RELEASE_REPLAY.md gains a section on data/uk/certified_pairs.json: the identity checks a pair shares with a variant, why no event replay runs on one, and why a pair stays once a variant arrives. ARCHITECTURE.md and the two lanes' annotation schemas name the new file where they named the variant index. Refs #228. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 3 +- docs/uk_replay/RELEASE_REPLAY.md | 28 +++++++++++++++++++ sources/cbp-rates-and-wrongs/annotations.json | 2 +- .../centax-childcare-100k/annotations.json | 2 +- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index db0d3e2b1..e1299e8de 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -240,7 +240,8 @@ Operating rules for the agent stage: come from `pipeline/compute_uk_childcare_100k.py` on the certified populace-uk pair: static, the three free-hours entitlements (plus Tax-Free Childcare for CBP) summed for people in English households. - Each certified variant's artifact keeps its run in the history, and a + Each certified variant's or certified pair's + (`data/uk/certified_pairs.json`) artifact keeps its run in the history, and a claim's latest result is the newest run that computed its year. Ingest takes an artifact only for a pair with a `verified_pairs` entry, and an engine or run note attaches only to the pairs its `variants` names. diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 42f3f6c4c..ee4ef7f47 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -181,6 +181,34 @@ A model/data combination is **incompatible** when no certified variant records it. There is no flag that relaxes a check: running a newer engine means adding a variant whose certification names it. +### Certified pairs (lane runs only) + +A lane can need a policyengine.py release's certified bundle before this +pipeline qualifies it as a variant: `uk-childcare-100k` (issue #228) runs on +policyengine.py 6.11.0's Microcosm pair. `data/uk/certified_pairs.json` +(kind `uk_certified_pairs`, `pipeline/uk_certified_pairs.py`) records each +such bundle in the variant record's shape less the fields that file a +replay: `pair_id` (`pair-`) in place of `variant_id`, status +`certified_pair`, and no `layout` or `result_root` +(`uk_certified_pairs.FIELDS` is the whole field set). + +- **The checks are the variant's.** `uk_replay_variants.validate_identity` + is check 1 for both kinds; `validate_record` adds only the variant's + status, layout, result root and legacy checks. A pair's preflight then + runs checks 2 to 6 through the same code: `verify_installed`, then + `compute_uk_event.materialized_preflight`, the second half of + `variant_preflight`. +- **No event replay runs on a pair.** The event runner resolves only + `replay_variants.json`; a lane resolves either kind with + `uk_certified_pairs.get_record`, and its ingest reads both files. +- **A pair stays when a variant arrives.** A variant this pipeline later + qualifies for the same bundle has its own requirements file, so its own + identity and id. The pair remains the identity the lane's runs on it were + checked against. +- Experiment populations (`docs/uk_replay/POLICY_EXPERIMENTS.md`) are a + third kind: their results are never ingested, while a lane's runs on a + pair are. + ## Unsupported years Each event registry's `calendar_years` is the construction's window; a diff --git a/sources/cbp-rates-and-wrongs/annotations.json b/sources/cbp-rates-and-wrongs/annotations.json index a6b1f1ee9..6e760d9d2 100644 --- a/sources/cbp-rates-and-wrongs/annotations.json +++ b/sources/cbp-rates-and-wrongs/annotations.json @@ -1,7 +1,7 @@ { "_schema": { "applies_to": "matcher on {models, periods, results}; null = any. models: endogenous_pension_exogenous_labour (Table 2), endogenous_labour_no_pension (Table 3); periods: claim periods (financial-year end years); results: certified", - "variants": "the certified variants (data/uk/replay_variants.json) on which a note resting on the engine or a run was checked; it attaches only to those pairs' results. A note resting on the report alone names none and applies to every run", + "variants": "the certified variants (data/uk/replay_variants.json) or certified pairs (data/uk/certified_pairs.json) on which a note resting on the engine or a run was checked; it attaches only to those pairs' results. A note resting on the report alone names none and applies to every run", "severity": "period | definitional | construction | caution | gap", "status": "the comparison status this annotation sets, where it sets one; at most one status-setting annotation matches a result", "basis": "traceability: raw/pages_layout.txt (PDF page), engine files at the certified pin, or the measured artifact results/uk/childcare_100k/.json" diff --git a/sources/centax-childcare-100k/annotations.json b/sources/centax-childcare-100k/annotations.json index c43a0b1dc..66f82ef95 100644 --- a/sources/centax-childcare-100k/annotations.json +++ b/sources/centax-childcare-100k/annotations.json @@ -1,7 +1,7 @@ { "_schema": { "applies_to": "matcher on {components, scenarios, periods, results}; null = any. components: Table 4.2 money columns (static_cost, intensive_margin, extensive_margin, net_fiscal_cost); scenarios: central, high_cost, low_cost; periods: claim periods (the tax year's later year); results: certified (the certified-pair run) or published_dashboard (the history row)", - "variants": "the certified variants (data/uk/replay_variants.json) on which a note resting on the engine or a run was checked; it attaches only to those pairs' results. A note resting on the report alone names none and applies to every run", + "variants": "the certified variants (data/uk/replay_variants.json) or certified pairs (data/uk/certified_pairs.json) on which a note resting on the engine or a run was checked; it attaches only to those pairs' results. A note resting on the report alone names none and applies to every run", "severity": "period | definitional | construction | caution | gap", "status": "the comparison status this annotation sets, where it sets one; at most one status-setting annotation matches a result", "basis": "traceability: raw/pages_layout.txt (PDF page and printed page), engine files at the certified pin, or the measured artifact results/uk/childcare_100k/.json" From 1f2ef0281599877adab109f1b61358ce01e68f1b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 06:56:11 -0700 Subject: [PATCH 3/5] Refuse a stripped pair id and a registry ingest would read differently Round-1 review of #229 (GPT-6.1 Sol, AGREE with two minor findings): - pairs.preflight refuses a pair view whose id has lost its pair- prefix, before verify_installed; one once reached the materialised path under a variant's id. - ingest's run_records refuses a repeated id in or across the two files, a pair id without pair- or a variant id with it, a pairs file of another kind, and an identity its digest does not seal. It used to keep the last of two rows and let a pair overwrite a variant, where the pipeline refused or resolved the variant. scorecard_db does not import pipeline/, so the digest rule is copied, and a differential property test pins it to uk_replay_variants.identity_sha256. The tests also cover what the review noted they did not: null identity leaves in the property test, refusal in a variant's form as well as a pair's, and whole-record agreement between ingest and the pipeline. Each of the seven registry mutations was accepted by the previous reader. Refs #228. Co-Authored-By: Claude Opus 5.5 --- pipeline/uk_certified_pairs.py | 9 +- scorecard_db/ingest_uk_childcare_100k.py | 57 +++++++- tests/test_uk_certified_pairs.py | 158 ++++++++++++++++++++--- 3 files changed, 201 insertions(+), 23 deletions(-) diff --git a/pipeline/uk_certified_pairs.py b/pipeline/uk_certified_pairs.py index 3aa64631c..20ca4af68 100644 --- a/pipeline/uk_certified_pairs.py +++ b/pipeline/uk_certified_pairs.py @@ -131,9 +131,12 @@ def preflight(record: dict, *, dataset_source: Path | None = None) -> dict: if not is_pair(record): return event.variant_preflight(record, dataset_source=dataset_source) - if record["status"] != STATUS: - raise variants.VariantError(f"{record['variant_id']}: not a certified pair") - variants.validate_identity(record, record["variant_id"].removeprefix(PREFIX)) + record_id = record.get("variant_id") + if record.get("status") != STATUS: + raise variants.VariantError(f"{record_id}: not a certified pair") + if not isinstance(record_id, str) or not record_id.startswith(PREFIX): + raise variants.VariantError(f"{record_id}: a pair's id starts {PREFIX}") + variants.validate_identity(record, record_id.removeprefix(PREFIX)) installed = variants.verify_installed(record) return event.materialized_preflight( record, installed, dataset_source=dataset_source diff --git a/scorecard_db/ingest_uk_childcare_100k.py b/scorecard_db/ingest_uk_childcare_100k.py index f3ebe97b5..5dc5bb099 100644 --- a/scorecard_db/ingest_uk_childcare_100k.py +++ b/scorecard_db/ingest_uk_childcare_100k.py @@ -76,6 +76,7 @@ from __future__ import annotations import datetime as dt +import hashlib import json import math import re @@ -104,6 +105,9 @@ RESULTS_DIR = REPO / "results" / "uk" / "childcare_100k" VARIANTS = REPO / "data" / "uk" / "replay_variants.json" PAIRS = REPO / "data" / "uk" / "certified_pairs.json" +# pipeline/uk_certified_pairs.py KIND and PREFIX (a test checks they agree) +PAIRS_KIND = "uk_certified_pairs" +PAIR_PREFIX = "pair-" CERTIFIED = REPO / "data" / "uk" / "certified_bundle.json" CONSTRUCTION = RESULTS_DIR / "construction_annotations.json" # The artifact schema this ingest reads (pipeline/compute_uk_childcare_100k.py). @@ -460,18 +464,63 @@ def check_artifact(art: dict, path: Path, variant: dict) -> None: raise ValueError(f"{path.name}: artifact is not a static run") +def identity_sha256(identity: dict) -> str: + """pipeline/uk_replay_variants.identity_sha256, which this package does + not import: SHA-256 of sorted-key compact JSON (a test checks the two + agree).""" + return hashlib.sha256( + json.dumps( + identity, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + ).hexdigest() + + def run_records(variants_path: Path = VARIANTS, pairs_path: Path = PAIRS) -> dict: """id -> record, for every record a run of this lane may name: each - certified replay variant, and each certified pair under its pair- id - (pipeline/uk_certified_pairs.py validates both files; a test checks the - committed ones).""" + certified replay variant, and each certified pair under its pair- id. + pipeline/uk_certified_pairs.py validates both files in full; reading + them here, ingest refuses what would let it and the pipeline resolve one + id to different records: a repeated id in or across the files, a pair id + without pair- or a variant id with it, and an identity its digest does + not seal.""" index = json.loads(variants_path.read_text()) + document = json.loads(pairs_path.read_text()) + if (document.get("schema_version"), document.get("kind")) != ( + 1, + PAIRS_KIND, + ): + raise ValueError(f"{pairs_path.name}: not a version 1 {PAIRS_KIND} file") + variant_ids = [v.get("variant_id") for v in index["variants"]] + pair_ids = [p.get("pair_id") for p in document["pairs"]] + repeated = sorted( + str(k) for k, n in Counter([*variant_ids, *pair_ids]).items() if n > 1 + ) + misnamed = sorted( + [str(v) for v in variant_ids if str(v).startswith(PAIR_PREFIX)] + + [str(p) for p in pair_ids if not str(p).startswith(PAIR_PREFIX)] + ) + if repeated or misnamed: + raise ValueError( + f"{variants_path.name} and {pairs_path.name}: repeated ids {repeated}, " + f"ids in the other file's namespace {misnamed}" + ) records = { v["variant_id"]: v for v in index["variants"] if v["status"] == "certified" } - for pair in json.loads(pairs_path.read_text())["pairs"]: + for pair in document["pairs"]: if pair.get("status") == "certified_pair": records[pair["pair_id"]] = {**pair, "variant_id": pair["pair_id"]} + unsealed = sorted( + k + for k, r in records.items() + if identity_sha256(r["identity"]) != r["identity_sha256"] + ) + if unsealed: + raise ValueError(f"identity_sha256 does not seal the identity of {unsealed}") return records diff --git a/tests/test_uk_certified_pairs.py b/tests/test_uk_certified_pairs.py index 25da4ed94..caa32548b 100644 --- a/tests/test_uk_certified_pairs.py +++ b/tests/test_uk_certified_pairs.py @@ -7,10 +7,14 @@ - every committed pair validates, under an id pair-, and no pair is a replay variant (neither file names the other's ids); - a pair binds its identity: changing any one value in it, without - resealing the digest, is refused (property test); + resealing the digest, is refused, and the same record in a variant's + form is refused too (property test); - a pair's checks are the replay variant's identity checks: given a variant's status, layout and result root, the same record passes validate_record (differential), and ingest reads the same records; +- ingest, which reads both files itself, refuses every registry the + pipeline's loaders refuse for a repeated or misplaced id, and its + identity digest is the pipeline's (differential, property test); - the preflight routes a pair through the pair checks and the materialised path, and a variant through the event runner's own preflight, never the other way round. @@ -99,16 +103,32 @@ def test_a_pair_is_refused_for_each_broken_field(change, message): def _leaves(value, path=()): - """(path, value) for every string or integer in a nested record.""" + """(path, value) for every string, integer or null in a nested record.""" if isinstance(value, dict): for key in sorted(value): yield from _leaves(value[key], (*path, key)) - elif isinstance(value, str) or ( - isinstance(value, int) and not isinstance(value, bool) + elif ( + value is None + or isinstance(value, str) + or (isinstance(value, int) and not isinstance(value, bool)) ): yield path, value +def _as_variant(record: dict) -> dict: + """The pair record with a replay variant's own fields in place of its + pair-only ones.""" + record = copy.deepcopy(record) + variant_id = record.pop("pair_id").removeprefix(pairs.PREFIX) + record.update( + variant_id=variant_id, + status="certified", + layout="replay_namespace", + result_root=f"{rv.REPLAY_RESULT_ROOT}/{variant_id}", + ) + return record + + _IDENTITY_LEAVES = list(_leaves(stored()["identity"])) @@ -119,30 +139,27 @@ def _leaves(value, path=()): ) def test_any_change_to_a_pairs_identity_is_refused(leaf, suffix): """The digest binds the identity: one changed value, with the stored - identity_sha256 kept, never validates.""" + identity_sha256 kept, never validates, as a pair or as a variant.""" path, value = leaf record = stored() node = record["identity"] for key in path[:-1]: node = node[key] - node[path[-1]] = value + 1 if isinstance(value, int) else value + suffix + if value is None: + node[path[-1]] = suffix + else: + node[path[-1]] = value + 1 if isinstance(value, int) else value + suffix with pytest.raises(rv.VariantError): pairs.validate_pair(record) + with pytest.raises(rv.VariantError): + rv.validate_record(_as_variant(record)) def test_a_pairs_checks_are_the_replay_variants_identity_checks(): """Differential: the stored pair, given a replay variant's own fields, is a valid variant record, so validate_identity is the whole of what the variant checks apply to it.""" - record = stored() - variant_id = record.pop("pair_id").removeprefix(pairs.PREFIX) - record.update( - variant_id=variant_id, - status="certified", - layout="replay_namespace", - result_root=f"{rv.REPLAY_RESULT_ROOT}/{variant_id}", - ) - rv.validate_record(record) + rv.validate_record(_as_variant(stored())) def test_ingest_reads_the_records_the_pipeline_resolves(): @@ -158,10 +175,113 @@ def test_ingest_reads_the_records_the_pipeline_resolves(): assert sorted(read) == sorted([*certified, *ids]) for record_id in read: resolved = pairs.get_record(record_id) - for field in ("identity", "identity_sha256", "data_years", "variant_id"): + for field in set(resolved) - {"certified_pair"}: assert read[record_id][field] == resolved[field], (record_id, field) +def test_ingest_names_the_pipelines_files_kind_and_prefix(): + assert (lane.VARIANTS, lane.PAIRS) == (rv.VARIANTS_PATH, pairs.PAIRS_PATH) + assert (lane.PAIRS_KIND, lane.PAIR_PREFIX) == (pairs.KIND, pairs.PREFIX) + + +_JSON_LEAF = st.none() | st.integers() | st.text(max_size=8) + + +@settings(max_examples=80, deadline=None) +@given( + identity=st.recursive( + st.dictionaries(st.text(max_size=6), _JSON_LEAF, max_size=4), + lambda children: st.dictionaries( + st.text(max_size=6), _JSON_LEAF | children, max_size=4 + ), + max_leaves=12, + ) +) +def test_ingest_seals_an_identity_as_the_pipeline_does(identity): + """Differential: scorecard_db's copy of the digest rule agrees with + pipeline/uk_replay_variants.identity_sha256 on any identity of strings, + integers and nulls, and on every committed record.""" + assert lane.identity_sha256(identity) == rv.identity_sha256(identity) + for record in lane.run_records().values(): + assert lane.identity_sha256(record["identity"]) == rv.identity_sha256( + record["identity"] + ) + + +def _registries(tmp_path, change): + """Copies of both committed files, changed by change(variants, pairs).""" + variants_doc = json.loads(rv.VARIANTS_PATH.read_text()) + pairs_doc = json.loads(pairs.PAIRS_PATH.read_text()) + change(variants_doc, pairs_doc) + v_path, p_path = tmp_path / "replay_variants.json", tmp_path / "pairs.json" + v_path.write_text(json.dumps(variants_doc)) + p_path.write_text(json.dumps(pairs_doc)) + return v_path, p_path + + +def _repeat_pair(v, p): + p["pairs"].append({**p["pairs"][0], "note": "a second row"}) + + +def _pair_takes_a_variant_id(v, p): + p["pairs"][0]["pair_id"] = v["variants"][-1]["variant_id"] + + +def _pair_without_prefix(v, p): + p["pairs"][0]["pair_id"] = p["pairs"][0]["pair_id"].removeprefix(pairs.PREFIX) + + +def _variant_with_prefix(v, p): + v["variants"][-1]["variant_id"] = pairs.PREFIX + v["variants"][-1]["variant_id"] + + +def _unsealed_pair(v, p): + p["pairs"][0]["identity"]["python"] = "3.13" + + +def _unsealed_variant(v, p): + v["variants"][-1]["identity"]["python"] = "3.13" + + +def _another_kind(v, p): + p["kind"] = "uk_replay_variants" + + +@pytest.mark.parametrize( + "change, message", + [ + (_repeat_pair, "repeated ids"), + (_pair_takes_a_variant_id, "repeated ids"), + (_pair_without_prefix, "other file's namespace"), + (_variant_with_prefix, "other file's namespace"), + (_unsealed_pair, "does not seal"), + (_unsealed_variant, "does not seal"), + (_another_kind, "not a version 1"), + ], +) +def test_ingest_refuses_a_registry_the_pipeline_would_read_otherwise( + tmp_path, change, message +): + """Round-1 review: ingest once kept the last of two rows with one id, + and let a pair id overwrite a variant's, where the pipeline refused the + file or resolved the variant. Every such registry is now refused, and + the pipeline refuses or resolves differently for each.""" + v_path, p_path = _registries(tmp_path, change) + with pytest.raises(ValueError, match=message): + lane.run_records(v_path, p_path) + first_pair = json.loads(p_path.read_text())["pairs"][0] + with pytest.raises(rv.VariantError): + if change in (_repeat_pair, _another_kind): + pairs.load_pairs(p_path) + elif change in (_variant_with_prefix, _unsealed_variant): + rv.get_variant( + json.loads(v_path.read_text())["variants"][-1]["variant_id"], + path=v_path, + ) + else: + pairs.validate_pair(first_pair) + + def test_the_preflight_routes_each_record_its_own_way(monkeypatch): calls = [] monkeypatch.setattr( @@ -199,6 +319,12 @@ def test_the_preflight_refuses_a_pair_view_changed_after_resolution(monkeypatch) pair["status"] = "certified" with pytest.raises(rv.VariantError, match="not a certified pair"): pairs.preflight(pair) + # round-1 review: a view stripped of its prefix once reached the + # materialised path under a variant's id + pair = pairs.get_record(PAIR) + pair["variant_id"] = pair["variant_id"].removeprefix(pairs.PREFIX) + with pytest.raises(rv.VariantError, match="a pair's id starts pair-"): + pairs.preflight(pair) def test_the_registry_refuses_a_repeated_id_and_another_kind(tmp_path): From 58e286fbb9470fc24ac0f938340de6c238716515 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 07:05:18 -0700 Subject: [PATCH 4/5] Re-export the feeds: two CBP rows cite the certified pairs file The 2031-32 CBP rows' pe_missing reason names both files a certified pair can come from (data/uk/replay_variants.json and data/uk/certified_pairs.json). That text, and the export date, are the only changes; the database content hash moves from e5497fd5 to 5fbff9ec for those two rows alone. No number changes. Refs #228. Co-Authored-By: Claude Opus 5.5 --- app/public/data/lanes.json | 2 +- app/public/data/populations.json | 6 +++--- data/lanes.json | 2 +- data/populations.json | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/public/data/lanes.json b/app/public/data/lanes.json index cb4e6cf39..f2e4c9dd4 100644 --- a/app/public/data/lanes.json +++ b/app/public/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK | BE | NZ \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-10-25", + "updated": "2026-10-26", "lanes": [ { "id": "urban-sotsn", diff --git a/app/public/data/populations.json b/app/public/data/populations.json index c3e16f1ff..83e10f984 100644 --- a/app/public/data/populations.json +++ b/app/public/data/populations.json @@ -1,5 +1,5 @@ { - "built": "2026-10-25", + "built": "2026-10-26", "note": "Non-Urban populations exported from scorecard.db: the populace reform-validation registry (issue #20) plus the compute campaign's attached comparisons (US: TPC/CPSP/PWBM/CBO/JCT; UK: the HMRC ready-reckoner family and the complete OBR Policy measures database, including historical HM Treasury scores; BE: JRC EUROMOD-BE claims with demo-grade Axiom concept-mismatch attachments and the 2026 PIT-reform scorekeeper claims; NZ: official budget scores). Claims explicitly opted into pre-result display remain in the feed with latest status not_computed and empty history. Recorded result statuses and calibration relationships are verbatim; nothing here is a pass/fail grade.", "summary": { "claims": 29771, @@ -30104,7 +30104,7 @@ "measure_key": "uk_free_childcare_hours_and_tfc_100k_limit_removed", "program": "free_childcare_hours_and_tax_free_childcare", "scoring_method": "post_behavioural", - "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json)" + "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json, data/uk/certified_pairs.json)" }, "measure": null, "reform_framework": "policy_ref", @@ -30161,7 +30161,7 @@ "measure_key": "uk_free_childcare_hours_and_tfc_100k_limit_removed", "program": "free_childcare_hours_and_tax_free_childcare", "scoring_method": "post_behavioural", - "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json)" + "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json, data/uk/certified_pairs.json)" }, "measure": null, "reform_framework": "policy_ref", diff --git a/data/lanes.json b/data/lanes.json index cb4e6cf39..f2e4c9dd4 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK | BE | NZ \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-10-25", + "updated": "2026-10-26", "lanes": [ { "id": "urban-sotsn", diff --git a/data/populations.json b/data/populations.json index c3e16f1ff..83e10f984 100644 --- a/data/populations.json +++ b/data/populations.json @@ -1,5 +1,5 @@ { - "built": "2026-10-25", + "built": "2026-10-26", "note": "Non-Urban populations exported from scorecard.db: the populace reform-validation registry (issue #20) plus the compute campaign's attached comparisons (US: TPC/CPSP/PWBM/CBO/JCT; UK: the HMRC ready-reckoner family and the complete OBR Policy measures database, including historical HM Treasury scores; BE: JRC EUROMOD-BE claims with demo-grade Axiom concept-mismatch attachments and the 2026 PIT-reform scorekeeper claims; NZ: official budget scores). Claims explicitly opted into pre-result display remain in the feed with latest status not_computed and empty history. Recorded result statuses and calibration relationships are verbatim; nothing here is a pass/fail grade.", "summary": { "claims": 29771, @@ -30104,7 +30104,7 @@ "measure_key": "uk_free_childcare_hours_and_tfc_100k_limit_removed", "program": "free_childcare_hours_and_tax_free_childcare", "scoring_method": "post_behavioural", - "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json)" + "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json, data/uk/certified_pairs.json)" }, "measure": null, "reform_framework": "policy_ref", @@ -30161,7 +30161,7 @@ "measure_key": "uk_free_childcare_hours_and_tfc_100k_limit_removed", "program": "free_childcare_hours_and_tax_free_childcare", "scoring_method": "post_behavioural", - "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json)" + "pe_missing": "fiscal year 2031-32 is outside the data years of every certified pair this lane has run (pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z 2023-2030; data/uk/replay_variants.json, data/uk/certified_pairs.json)" }, "measure": null, "reform_framework": "policy_ref", From 466df5afddef086db684e81da15bbb541b33c556 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 09:54:23 -0700 Subject: [PATCH 5/5] Close the round-2 review nits on the pair registry GPT-6.1 Sol round 2 of #229 (AGREE, two nits): - run_records refuses a variants file whose schema_version is not 1, as the pipeline's loader does; the registry-mutation test gains the case. - The digest differential draws floats, NaN and infinity included, and compares the two rules' digest or refusal; an allow_nan=True copy, which the round-2 mutants showed surviving, now fails it. Also sets deadline=None on test_no_local_data_manifest_survives_isolation, as its neighbours have: the review's first run under host load failed it on Hypothesis's 200 ms deadline (319 ms), and it passed alone. Refs #228. Co-Authored-By: Claude Opus 5.5 --- scorecard_db/ingest_uk_childcare_100k.py | 2 ++ tests/test_uk_certified_pairs.py | 27 ++++++++++++++++++++---- tests/test_uk_replay_variants.py | 3 +++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scorecard_db/ingest_uk_childcare_100k.py b/scorecard_db/ingest_uk_childcare_100k.py index 5dc5bb099..ac5824be2 100644 --- a/scorecard_db/ingest_uk_childcare_100k.py +++ b/scorecard_db/ingest_uk_childcare_100k.py @@ -488,6 +488,8 @@ def run_records(variants_path: Path = VARIANTS, pairs_path: Path = PAIRS) -> dic without pair- or a variant id with it, and an identity its digest does not seal.""" index = json.loads(variants_path.read_text()) + if index.get("schema_version") != 1: + raise ValueError(f"{variants_path.name}: schema_version must be 1") document = json.loads(pairs_path.read_text()) if (document.get("schema_version"), document.get("kind")) != ( 1, diff --git a/tests/test_uk_certified_pairs.py b/tests/test_uk_certified_pairs.py index caa32548b..50358c7f2 100644 --- a/tests/test_uk_certified_pairs.py +++ b/tests/test_uk_certified_pairs.py @@ -184,10 +184,19 @@ def test_ingest_names_the_pipelines_files_kind_and_prefix(): assert (lane.PAIRS_KIND, lane.PAIR_PREFIX) == (pairs.KIND, pairs.PREFIX) -_JSON_LEAF = st.none() | st.integers() | st.text(max_size=8) +# floats too, NaN and infinity included: the pipeline refuses those +# (allow_nan=False), and a copy that allowed them must not pass +_JSON_LEAF = st.none() | st.integers() | st.floats() | st.text(max_size=8) -@settings(max_examples=80, deadline=None) +def _digest_or_refusal(rule, identity): + try: + return rule(identity) + except ValueError: + return ValueError + + +@settings(max_examples=120, deadline=None) @given( identity=st.recursive( st.dictionaries(st.text(max_size=6), _JSON_LEAF, max_size=4), @@ -200,8 +209,11 @@ def test_ingest_names_the_pipelines_files_kind_and_prefix(): def test_ingest_seals_an_identity_as_the_pipeline_does(identity): """Differential: scorecard_db's copy of the digest rule agrees with pipeline/uk_replay_variants.identity_sha256 on any identity of strings, - integers and nulls, and on every committed record.""" - assert lane.identity_sha256(identity) == rv.identity_sha256(identity) + integers, floats and nulls (both refuse NaN and infinity), and on every + committed record.""" + assert _digest_or_refusal(lane.identity_sha256, identity) == _digest_or_refusal( + rv.identity_sha256, identity + ) for record in lane.run_records().values(): assert lane.identity_sha256(record["identity"]) == rv.identity_sha256( record["identity"] @@ -247,6 +259,10 @@ def _another_kind(v, p): p["kind"] = "uk_replay_variants" +def _another_variants_version(v, p): + v["schema_version"] = 2 + + @pytest.mark.parametrize( "change, message", [ @@ -257,6 +273,7 @@ def _another_kind(v, p): (_unsealed_pair, "does not seal"), (_unsealed_variant, "does not seal"), (_another_kind, "not a version 1"), + (_another_variants_version, "schema_version must be 1"), ], ) def test_ingest_refuses_a_registry_the_pipeline_would_read_otherwise( @@ -273,6 +290,8 @@ def test_ingest_refuses_a_registry_the_pipeline_would_read_otherwise( with pytest.raises(rv.VariantError): if change in (_repeat_pair, _another_kind): pairs.load_pairs(p_path) + elif change is _another_variants_version: + rv.load_document(v_path) elif change in (_variant_with_prefix, _unsealed_variant): rv.get_variant( json.loads(v_path.read_text())["variants"][-1]["variant_id"], diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py index a80f89c52..1f7d61891 100644 --- a/tests/test_uk_replay_variants.py +++ b/tests/test_uk_replay_variants.py @@ -316,6 +316,9 @@ def test_certification_is_isolated_from_a_local_data_manifest(): assert environ["PATH"] == "/bin" and environ["HF_HUB_OFFLINE"] == "1" +# deadline=None like its neighbours: under host load a first example once +# took 319 ms against Hypothesis's 200 ms default (#229 round-2 review) +@settings(deadline=None) @given( country=st.text( alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", min_size=1, max_size=6