From 2fc3a3d0176e59758c7bb5905e38f971dcde1d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:15:43 +0000 Subject: [PATCH 01/18] Resolve the current build a UK cut compares with A UK cut will check and certify its candidate against the current build of its own release line, as the Hub's releases.json names it (microcosm#1095), instead of the enhanced FRS. This adds the resolution, before any check uses it. - microcosm-data's loader exposes CertifiedRelease, resolve_certified, download_certified and download_build_file: the build-folder resolution it already does for load(), returning the build's record, without enforcing the installed engine's versions, since a comparison reads a build under the engine it runs. - uk_runtime/cut_reference.py resolves the national or dense line's current build, checks its certification evidence by its fields (the fetch runs without the signing key), and keeps a strict record that later steps read instead of resolving again. - tools/resolve_uk_reference.py checks a line's current build read-only, or fetches it with its record for runs outside run_uk_cut. Co-Authored-By: Claude Opus 5.5 --- .../build/uk_runtime/cut_reference.py | 422 ++++++++++++++++++ .../uk/test_resolve_uk_reference.py | 80 ++++ .../engine_free/uk/test_uk_cut_reference.py | 218 +++++++++ .../src/microcosm/data/loader.py | 129 +++++- .../tests/engine_free/shared/test_loader.py | 107 +++++ .../microcosm_build/uk_cut_reference.py | 187 ++++++++ tools/resolve_uk_reference.py | 94 ++++ 7 files changed, 1234 insertions(+), 3 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_cut_reference.py create mode 100644 test_support/microcosm_build/uk_cut_reference.py create mode 100644 tools/resolve_uk_reference.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py new file mode 100644 index 0000000000..ce1a803ecc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py @@ -0,0 +1,422 @@ +"""The current build a UK cut is checked against (microcosm#1095). + +A cut compares its candidate with the current build of its own release line, +as the Hub's ``releases.json`` names it when the cut fetches its inputs: the +national line with the national, the dense line with the dense. Comparing +the dense file with the national would mix the lines' differences (clones, +local targets, area columns) into checks meant to catch change between cuts. + +The reference is resolved once per cut. The fetch writes its record, and +every later step reads that record and re-checks the file's SHA-256 against +it; none re-resolves, so a promotion while a cut runs cannot split its +reference. Each check that uses the reference names it in its own output. + +The fetch runs without the gate signing key, so it reads a build's +certification by its fields, not its signature: a national build's +``release_certification.json`` must be shippable, name this file's SHA-256 +and carry a registered key's fingerprint, and a dense build's +``gate_summary.json`` must report every gate passed. The publication that +made the build current verified the signatures. ``releases.json`` itself +never names a withdrawn build as current. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from microcosm.build.uk_runtime.release_identity import ( + UK_DENSE_RELEASE_ID, + UK_NATIONAL_RELEASE_ID, +) + +__all__ = [ + "UK_CUT_REFERENCE_KIND", + "UK_CUT_REFERENCE_RELEASES", + "UK_CUT_REFERENCE_SCHEMA_VERSION", + "UKCutReference", + "check_uk_cut_reference", + "read_uk_cut_reference", + "resolve_uk_cut_reference", + "verify_uk_cut_reference_file", + "write_uk_cut_reference", +] + +UK_CUT_REFERENCE_KIND = "uk_cut_reference" +UK_CUT_REFERENCE_SCHEMA_VERSION = 1 + +#: Each line's registry variant and the build evidence its fetch reads. +UK_CUT_REFERENCE_RELEASES: Mapping[str, tuple[str, str]] = { + UK_NATIONAL_RELEASE_ID: ("compact", "release_certification"), + UK_DENSE_RELEASE_ID: ("dense", "gate_summary"), +} +_REGISTRY_YEAR = 2025 + +_HubDownload = Callable[..., str] + +_RECORD_KEYS = frozenset( + { + "kind", + "schema_version", + "release", + "build", + "repo_id", + "path_in_repo", + "filename", + "sha256", + "size_bytes", + "built_with_model_package", + "built_with_core_package", + "evidence", + "resolved_at", + } +) + + +@dataclass(frozen=True) +class UKCutReference: + """One build of a UK release line, resolved for a cut to compare with.""" + + release: str + build: str + repo_id: str + path_in_repo: str + filename: str + sha256: str + size_bytes: int + built_with_model_package: Mapping[str, str] + built_with_core_package: Mapping[str, str] + evidence: Mapping[str, str] + resolved_at: str + + @property + def label(self) -> str: + """``@``, the name every check that uses it records.""" + + return f"{self.release}@{self.build}" + + def to_record(self) -> dict[str, Any]: + return { + "kind": UK_CUT_REFERENCE_KIND, + "schema_version": UK_CUT_REFERENCE_SCHEMA_VERSION, + "release": self.release, + "build": self.build, + "repo_id": self.repo_id, + "path_in_repo": self.path_in_repo, + "filename": self.filename, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "built_with_model_package": dict(self.built_with_model_package), + "built_with_core_package": dict(self.built_with_core_package), + "evidence": dict(self.evidence), + "resolved_at": self.resolved_at, + } + + @classmethod + def from_record(cls, payload: object) -> UKCutReference: + if not isinstance(payload, Mapping) or set(payload) != _RECORD_KEYS: + raise ValueError( + "A UK cut reference record must have exactly the keys " + f"{sorted(_RECORD_KEYS)}." + ) + if payload["kind"] != UK_CUT_REFERENCE_KIND: + raise ValueError( + f"A UK cut reference record has kind {UK_CUT_REFERENCE_KIND!r}, " + f"not {payload['kind']!r}." + ) + if payload["schema_version"] != UK_CUT_REFERENCE_SCHEMA_VERSION: + raise ValueError( + "A UK cut reference record has schema_version " + f"{UK_CUT_REFERENCE_SCHEMA_VERSION}, not " + f"{payload['schema_version']!r}." + ) + if payload["release"] not in UK_CUT_REFERENCE_RELEASES: + raise ValueError( + f"A UK cut reference names release {payload['release']!r}; a " + f"cut compares with one of {sorted(UK_CUT_REFERENCE_RELEASES)}." + ) + for key in ("build", "repo_id", "path_in_repo", "filename", "resolved_at"): + if not isinstance(payload[key], str) or not payload[key]: + raise ValueError(f"A UK cut reference record has no {key}.") + sha256 = payload["sha256"] + if not _is_sha256(sha256): + raise ValueError("A UK cut reference record has no valid sha256.") + size = payload["size_bytes"] + if type(size) is not int or size <= 0: + raise ValueError("A UK cut reference record has no positive size_bytes.") + for key in ("built_with_model_package", "built_with_core_package"): + package = payload[key] + if ( + not isinstance(package, Mapping) + or set(package) != {"name", "version"} + or not all( + isinstance(value, str) and value for value in package.values() + ) + ): + raise ValueError( + f"A UK cut reference record's {key} must name a package and " + "its version." + ) + evidence = payload["evidence"] + if ( + not isinstance(evidence, Mapping) + or set(evidence) != {"key", "filename", "sha256"} + or not _is_sha256(evidence["sha256"]) + ): + raise ValueError( + "A UK cut reference record's evidence must name the build file " + "its fetch read, with its sha256." + ) + return cls( + release=payload["release"], + build=payload["build"], + repo_id=payload["repo_id"], + path_in_repo=payload["path_in_repo"], + filename=payload["filename"], + sha256=sha256, + size_bytes=size, + built_with_model_package=dict(payload["built_with_model_package"]), + built_with_core_package=dict(payload["built_with_core_package"]), + evidence=dict(evidence), + resolved_at=payload["resolved_at"], + ) + + +def _is_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _line(release: str) -> tuple[str, str]: + try: + return UK_CUT_REFERENCE_RELEASES[release] + except KeyError: + raise ValueError( + f"A UK cut compares with the current build of {release!r} only if " + f"it is one of {sorted(UK_CUT_REFERENCE_RELEASES)}." + ) from None + + +def _check_national_certification( + payload: object, *, release: str, build: str, dataset_sha256: str +) -> None: + from microcosm.data.gate_signing_keys import registered_gate_signing_key + + where = f"release_certification.json of {release}@{build}" + if not isinstance(payload, Mapping): + raise ValueError(f"The {where} is not a JSON object.") + if payload.get("kind") != "uk_release_certification": + raise ValueError(f"The {where} is not a UK release certification.") + if payload.get("release_id") != release: + raise ValueError(f"The {where} certifies another release.") + if payload.get("shippable") is not True: + raise ValueError( + f"The current build {release}@{build} is not certified shippable; a " + "cut compares only with a certified build." + ) + candidate = payload.get("candidate") + if not isinstance(candidate, Mapping) or candidate.get("sha256") != dataset_sha256: + raise ValueError( + f"The {where} certifies another file than the build's dataset." + ) + attestation = payload.get("attestation") + fingerprint = ( + attestation.get("signing_key_sha256") + if isinstance(attestation, Mapping) + else None + ) + if not isinstance(fingerprint, str) or ( + registered_gate_signing_key("uk", fingerprint) is None + ): + raise ValueError(f"The {where} was not signed by a registered UK gate key.") + + +def _check_dense_gate_summary(payload: object, *, release: str, build: str) -> None: + where = f"gate_summary.json of {release}@{build}" + if not isinstance(payload, Mapping): + raise ValueError(f"The {where} is not a JSON object.") + gates = payload.get("gates") + if not isinstance(gates, Mapping) or not gates: + raise ValueError(f"The {where} reports no gates.") + failing = sorted( + name + for name, gate in gates.items() + if not isinstance(gate, Mapping) + or gate.get("passed") is not True + or gate.get("status") != "passed" + ) + if failing: + raise ValueError( + f"The current build {release}@{build} did not pass " + f"{', '.join(failing)}; a cut compares only with a certified build." + ) + + +def _checked_evidence( + certified: Any, + *, + release: str, + evidence_key: str, + hub_download: _HubDownload | None, +) -> dict[str, str]: + from microcosm.data.loader import download_build_file + + path = download_build_file(certified, evidence_key, hub_download=hub_download) + try: + payload = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise ValueError( + f"Could not read {path.name} of {release}@{certified.build}: {exc}." + ) from exc + if evidence_key == "release_certification": + _check_national_certification( + payload, + release=release, + build=certified.build, + dataset_sha256=certified.artifact_sha256, + ) + else: + _check_dense_gate_summary(payload, release=release, build=certified.build) + return {"key": evidence_key, "filename": path.name, "sha256": _sha256(path)} + + +def check_uk_cut_reference( + release: str, *, hub_download: _HubDownload | None = None +) -> dict[str, str]: + """Which build a cut of ``release`` would compare with now, read-only. + + Reads the release record, the build's manifest and its certification + evidence, with every check the fetch makes, but downloads no dataset. + """ + + from microcosm.data.loader import resolve_certified + + variant, evidence_key = _line(release) + certified = resolve_certified( + "uk", _REGISTRY_YEAR, variant=variant, hub_download=hub_download + ) + _require_build_folder(certified, release=release) + evidence = _checked_evidence( + certified, release=release, evidence_key=evidence_key, hub_download=hub_download + ) + return { + "release": release, + "build": certified.build, + "path_in_repo": certified.artifact_path, + "sha256": certified.artifact_sha256, + "built_with_model_package": ( + f"{certified.model.name} {certified.model.built_version}" + ), + "evidence": evidence["filename"], + } + + +def _require_build_folder(certified: Any, *, release: str) -> None: + if certified.release_id != release or certified.build is None: + raise ValueError( + f"{release} is not recorded in the Hub's releases.json, so it has no " + "current build for a cut to compare with." + ) + + +def resolve_uk_cut_reference( + release: str, + *, + hub_download: _HubDownload | None = None, + now: Callable[[], datetime] | None = None, +) -> tuple[Path, UKCutReference]: + """Download and check the current build of ``release`` for a cut. + + Returns the verified dataset's local path and the record a cut keeps. + Refuses a release the Hub does not record in ``releases.json`` and a + current build whose certification evidence does not pass. + """ + + from microcosm.data.loader import download_certified + + variant, evidence_key = _line(release) + path, certified = download_certified( + "uk", _REGISTRY_YEAR, variant=variant, hub_download=hub_download + ) + _require_build_folder(certified, release=release) + evidence = _checked_evidence( + certified, release=release, evidence_key=evidence_key, hub_download=hub_download + ) + resolved_at = (now or (lambda: datetime.now(UTC)))().strftime("%Y-%m-%dT%H:%M:%SZ") + reference = UKCutReference( + release=release, + build=certified.build, + repo_id=certified.artifact_repo_id, + path_in_repo=certified.artifact_path, + filename=Path(certified.artifact_path).name, + sha256=certified.artifact_sha256, + size_bytes=path.stat().st_size, + built_with_model_package={ + "name": certified.model.name, + "version": certified.model.built_version, + }, + built_with_core_package={ + "name": certified.core.name, + "version": certified.core.built_version, + }, + evidence=evidence, + resolved_at=resolved_at, + ) + return path, reference + + +def write_uk_cut_reference(path: str | Path, reference: UKCutReference) -> Path: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(reference.to_record(), indent=2, sort_keys=True)) + return destination + + +def read_uk_cut_reference(path: str | Path) -> UKCutReference: + source = Path(path) + try: + payload = json.loads(source.read_text()) + except (OSError, ValueError) as exc: + raise ValueError( + f"Could not read the cut's reference record {source}: {exc}." + ) from exc + return UKCutReference.from_record(payload) + + +def verify_uk_cut_reference_file(reference: UKCutReference, path: str | Path) -> Path: + """Refuse ``path`` unless it is the file ``reference`` records.""" + + source = Path(path) + if not source.is_file(): + raise ValueError( + f"The cut's reference {reference.label} is not at {source}; fetch it again." + ) + size = source.stat().st_size + if size != reference.size_bytes: + raise ValueError( + f"{source} is {size} bytes, but the cut's reference {reference.label} " + f"is {reference.size_bytes}." + ) + observed = _sha256(source) + if observed != reference.sha256: + raise ValueError( + f"{source} has sha256 {observed}, but the cut's reference " + f"{reference.label} has {reference.sha256}." + ) + return source diff --git a/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py b/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py new file mode 100644 index 0000000000..9b19c3fc2f --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py @@ -0,0 +1,80 @@ +"""tools/resolve_uk_reference.py: a line's current build, checked or fetched.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from microcosm.build.uk_runtime.cut_reference import read_uk_cut_reference +from test_support.microcosm_build.uk_cut_reference import ( + UK_DENSE_RELEASE, + UK_NATIONAL_RELEASE, + UKReferenceHub, + sha256_of, +) +from tools import resolve_uk_reference as tool + +DENSE_BUILD = "uk-local-candidate-f100-s42-20261107T173210Z-38605444" +NATIONAL_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" + + +@pytest.fixture +def hub(tmp_path: Path) -> UKReferenceHub: + hub = UKReferenceHub(tmp_path / "hub-root") + hub.publish(UK_NATIONAL_RELEASE, NATIONAL_BUILD, dataset=b"national") + hub.publish(UK_DENSE_RELEASE, DENSE_BUILD, dataset=b"dense") + return hub + + +def test_check_prints_the_current_build_and_downloads_no_dataset(hub, capsys) -> None: + assert tool.main(["--release", UK_DENSE_RELEASE, "--check"], hub_download=hub) == 0 + + summary = json.loads(capsys.readouterr().out) + assert summary["build"] == DENSE_BUILD + assert not any(call.endswith(".h5") for call in hub.calls) + + +def test_output_dir_keeps_the_dataset_and_its_record(hub, tmp_path: Path) -> None: + out = tmp_path / "reference" + + assert ( + tool.main( + ["--release", UK_DENSE_RELEASE, "--output-dir", str(out)], hub_download=hub + ) + == 0 + ) + + reference = read_uk_cut_reference(out / "reference.json") + assert reference.build == DENSE_BUILD + assert (out / reference.filename).read_bytes() == b"dense" + assert reference.sha256 == sha256_of(b"dense") + + +def test_a_directory_that_holds_a_reference_is_refused( + hub, tmp_path: Path, capsys +) -> None: + out = tmp_path / "reference" + args = ["--release", UK_NATIONAL_RELEASE, "--output-dir", str(out)] + assert tool.main(args, hub_download=hub) == 0 + + assert tool.main(args, hub_download=hub) == 1 + assert "keeps the reference it started with" in capsys.readouterr().err + + +def test_a_release_without_a_certified_current_build_is_refused( + tmp_path: Path, capsys +) -> None: + hub = UKReferenceHub(tmp_path) + hub.publish( + UK_NATIONAL_RELEASE, + NATIONAL_BUILD, + dataset=b"national", + evidence={"kind": "uk_release_certification", "shippable": False}, + ) + + assert ( + tool.main(["--release", UK_NATIONAL_RELEASE, "--check"], hub_download=hub) == 1 + ) + assert "refused" in capsys.readouterr().err diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cut_reference.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cut_reference.py new file mode 100644 index 0000000000..503b1fe858 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cut_reference.py @@ -0,0 +1,218 @@ +"""A UK cut's reference: the current build of its own release line (#1095).""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from microcosm.build.uk_runtime.cut_reference import ( + UKCutReference, + check_uk_cut_reference, + read_uk_cut_reference, + resolve_uk_cut_reference, + verify_uk_cut_reference_file, + write_uk_cut_reference, +) +from test_support.microcosm_build.uk_cut_reference import ( + UK_DENSE_RELEASE, + UK_NATIONAL_RELEASE, + UK_REPO_ID, + UKReferenceHub, + dense_gate_summary, + national_certification, + sha256_of, +) + +NATIONAL_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +EARLIER_NATIONAL_BUILD = "uk-frs-calibration-attempt-20261025T091414Z-a6299504" +DENSE_BUILD = "uk-local-candidate-f100-s42-20261107T173210Z-38605444" +NATIONAL_BYTES = b"national current build" +DENSE_BYTES = b"dense current build" + + +def _fixed_now() -> datetime: + return datetime(2026, 11, 10, 12, 0, tzinfo=UTC) + + +@pytest.fixture +def hub(tmp_path: Path) -> UKReferenceHub: + hub = UKReferenceHub(tmp_path) + hub.publish(UK_NATIONAL_RELEASE, EARLIER_NATIONAL_BUILD, dataset=b"older") + hub.publish(UK_NATIONAL_RELEASE, NATIONAL_BUILD, dataset=NATIONAL_BYTES) + hub.publish(UK_DENSE_RELEASE, DENSE_BUILD, dataset=DENSE_BYTES) + return hub + + +def test_a_national_cut_compares_with_the_national_current_build(hub) -> None: + path, reference = resolve_uk_cut_reference( + UK_NATIONAL_RELEASE, hub_download=hub, now=_fixed_now + ) + + assert path.read_bytes() == NATIONAL_BYTES + assert reference.release == UK_NATIONAL_RELEASE + assert reference.build == NATIONAL_BUILD + assert reference.label == f"{UK_NATIONAL_RELEASE}@{NATIONAL_BUILD}" + assert reference.repo_id == UK_REPO_ID + assert reference.path_in_repo == ( + f"releases/{UK_NATIONAL_RELEASE}/{NATIONAL_BUILD}/microcosm_uk_2024_25.h5" + ) + assert reference.filename == "microcosm_uk_2024_25.h5" + assert reference.sha256 == sha256_of(NATIONAL_BYTES) + assert reference.size_bytes == len(NATIONAL_BYTES) + assert reference.built_with_model_package == { + "name": "policyengine-uk", + "version": "2.174.1", + } + assert reference.evidence["key"] == "release_certification" + assert reference.resolved_at == "2026-11-10T12:00:00Z" + + +def test_a_dense_cut_compares_with_the_dense_current_build(hub) -> None: + path, reference = resolve_uk_cut_reference(UK_DENSE_RELEASE, hub_download=hub) + + assert path.read_bytes() == DENSE_BYTES + assert reference.build == DENSE_BUILD + assert reference.filename == "microcosm_uk_2024_25_dense.h5" + assert reference.evidence["key"] == "gate_summary" + + +def test_the_check_names_the_build_without_downloading_its_dataset(hub) -> None: + summary = check_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + + assert summary["build"] == NATIONAL_BUILD + assert summary["sha256"] == sha256_of(NATIONAL_BYTES) + assert not any(call.endswith(".h5") for call in hub.calls) + + +def test_a_promotion_moves_what_the_next_fetch_resolves(hub) -> None: + hub.publish( + UK_NATIONAL_RELEASE, + "uk-frs-calibration-attempt-20261112T000000Z-0new0000", + dataset=b"newer", + ) + + _, reference = resolve_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + + assert reference.build == "uk-frs-calibration-attempt-20261112T000000Z-0new0000" + + +@pytest.mark.parametrize( + ("evidence", "message"), + [ + ( + national_certification(NATIONAL_BYTES, shippable=False), + "not certified shippable", + ), + ( + national_certification(NATIONAL_BYTES, candidate={"sha256": "0" * 64}), + "certifies another file", + ), + ( + national_certification( + NATIONAL_BYTES, attestation={"signing_key_sha256": "1" * 64} + ), + "registered UK gate key", + ), + ( + national_certification(NATIONAL_BYTES, release_id=UK_DENSE_RELEASE), + "certifies another release", + ), + ], +) +@pytest.mark.committed_gate_key_registry +def test_a_national_build_without_a_shippable_certification_is_refused( + tmp_path: Path, evidence, message +) -> None: + hub = UKReferenceHub(tmp_path) + hub.publish( + UK_NATIONAL_RELEASE, NATIONAL_BUILD, dataset=NATIONAL_BYTES, evidence=evidence + ) + + with pytest.raises(ValueError, match=message): + resolve_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + + +def test_a_dense_build_with_a_failed_gate_is_refused(tmp_path: Path) -> None: + hub = UKReferenceHub(tmp_path) + hub.publish( + UK_DENSE_RELEASE, + DENSE_BUILD, + dataset=DENSE_BYTES, + evidence=dense_gate_summary( + uk_local_target_fit={"failures": ["x"], "passed": False, "status": "failed"} + ), + ) + + with pytest.raises(ValueError, match="did not pass uk_local_target_fit"): + resolve_uk_cut_reference(UK_DENSE_RELEASE, hub_download=hub) + + +def test_a_release_without_a_current_build_is_refused(tmp_path: Path) -> None: + hub = UKReferenceHub(tmp_path) + hub.publish( + UK_NATIONAL_RELEASE, NATIONAL_BUILD, dataset=NATIONAL_BYTES, current=False + ) + + with pytest.raises(ValueError, match="has no current build"): + resolve_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + + +def test_a_withdrawn_current_build_is_refused(tmp_path: Path) -> None: + hub = UKReferenceHub(tmp_path) + hub.publish(UK_NATIONAL_RELEASE, NATIONAL_BUILD, dataset=NATIONAL_BYTES) + record = json.loads((tmp_path / "hub" / "releases.json").read_text()) + record["releases"][UK_NATIONAL_RELEASE]["builds"][NATIONAL_BUILD]["withdrawn"] = ( + True + ) + hub.add_file("releases.json", json.dumps(record).encode()) + + with pytest.raises(ValueError, match="withdrawn"): + resolve_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + + +def test_a_line_outside_the_cut_lines_is_refused(hub) -> None: + with pytest.raises(ValueError, match="compares with the current build"): + resolve_uk_cut_reference("populace-uk-2023", hub_download=hub) + + +def test_the_record_round_trips_and_pins_the_file(hub, tmp_path: Path) -> None: + path, reference = resolve_uk_cut_reference( + UK_NATIONAL_RELEASE, hub_download=hub, now=_fixed_now + ) + record_path = write_uk_cut_reference(tmp_path / "cut" / "reference.json", reference) + + assert read_uk_cut_reference(record_path) == reference + assert verify_uk_cut_reference_file(reference, path) == path + + other = tmp_path / "other.h5" + other.write_bytes(b"x" * len(NATIONAL_BYTES)) + with pytest.raises(ValueError, match="has sha256"): + verify_uk_cut_reference_file(reference, other) + with pytest.raises(ValueError, match="is not at"): + verify_uk_cut_reference_file(reference, tmp_path / "missing.h5") + + +@pytest.mark.parametrize( + "mutate", + [ + lambda record: record.pop("evidence"), + lambda record: record.update(kind="something_else"), + lambda record: record.update(schema_version=2), + lambda record: record.update(sha256="not-a-digest"), + lambda record: record.update(size_bytes=0), + lambda record: record.update(release="populace-uk-2023"), + lambda record: record.update( + built_with_model_package={"name": "policyengine-uk"} + ), + ], +) +def test_a_malformed_record_is_refused(hub, mutate) -> None: + _, reference = resolve_uk_cut_reference(UK_NATIONAL_RELEASE, hub_download=hub) + record = reference.to_record() + mutate(record) + + with pytest.raises(ValueError): + UKCutReference.from_record(record) diff --git a/packages/microcosm-data/src/microcosm/data/loader.py b/packages/microcosm-data/src/microcosm/data/loader.py index ed61d2dd00..e5535d85e4 100644 --- a/packages/microcosm-data/src/microcosm/data/loader.py +++ b/packages/microcosm-data/src/microcosm/data/loader.py @@ -63,12 +63,16 @@ ) __all__ = [ + "CertifiedRelease", "available", "available_variants", "resolve", "latest_year", "download", + "download_build_file", + "download_certified", "load", + "resolve_certified", ] _SHA256_PATTERN = re.compile(r"[0-9a-fA-F]{64}") @@ -99,7 +103,7 @@ class _PackageCertification: @dataclass(frozen=True) -class _CertifiedRelease: +class CertifiedRelease: """A release's certified dataset: where it is and what it was built with. For a build-folder release, ``build`` is the build that was loaded, @@ -117,6 +121,9 @@ class _CertifiedRelease: build: str | None = None +_CertifiedRelease = CertifiedRelease + + class _CertifiedPackageCompatibilityError(ValueError): """Installed package versions do not satisfy a release certificate.""" @@ -707,9 +714,13 @@ def _resolve_certified_release( def _verified_download( - spec: DatasetSpec, *, build: str | None = None + spec: DatasetSpec, + *, + build: str | None = None, + hub_download: _HubDownload | None = None, ) -> tuple[Path, _CertifiedRelease]: - hub_download = _hub_download() + if hub_download is None: + hub_download = _hub_download() certified = _resolve_certified_release(spec, hub_download=hub_download, build=build) artifact_path = Path( hub_download( @@ -765,6 +776,118 @@ def download( return path +def resolve_certified( + country: str, + year: int | None = None, + *, + variant: str = DEFAULT_VARIANT, + build: str | None = None, + hub_download: _HubDownload | None = None, +) -> CertifiedRelease: + """Say which build :func:`download_certified` would download, without it. + + Reads only the release record and the build's manifest, with the same + checks; the dataset itself is neither downloaded nor verified. + """ + spec = resolve(country, year, variant=variant) + return _resolve_certified_release(spec, hub_download=hub_download, build=build) + + +def download_certified( + country: str, + year: int | None = None, + *, + variant: str = DEFAULT_VARIANT, + build: str | None = None, + hub_download: _HubDownload | None = None, +) -> tuple[Path, CertifiedRelease]: + """Download and SHA-verify a certified dataset, and say which build it is. + + The same resolution and checks as :func:`download`, returning with the + path the build's record: its release, build, artifact path and SHA-256, + and the model and Core versions it was built with. Unlike :func:`load`, + it neither enforces the installed engine's versions nor constructs a + dataset, so a comparison can read a build under the engine it runs (a UK + cut compares its candidate with the current build this way). + + Raises: + ImportError: If ``huggingface_hub`` is not installed. + ValueError: If release metadata or the artifact digest is invalid. + """ + spec = resolve(country, year, variant=variant) + return _verified_download(spec, build=build, hub_download=hub_download) + + +def download_build_file( + certified: CertifiedRelease, + key: str, + *, + hub_download: _HubDownload | None = None, +) -> Path: + """Download one file a build's release manifest declares, SHA-verified. + + ``key`` names an entry of the build's ``release_manifest.json`` + ``artifacts``; the file is read from the build's own folder, which never + changes after publishing. + + Raises: + ValueError: If the release keeps the tagged layout (no build folder), + the manifest declares no such artifact, or its digest differs. + """ + if certified.build is None: + raise ValueError( + f"Release {certified.release_id!r} keeps the tagged layout; only a " + "build published into its own folder declares its files there." + ) + if hub_download is None: + hub_download = _hub_download() + folder = build_folder(certified.release_id, certified.build) + manifest = _load_manifest( + Path( + hub_download( + repo_id=certified.artifact_repo_id, + filename=f"{folder}/release_manifest.json", + repo_type="dataset", + ) + ), + repo_id=certified.artifact_repo_id, + release_id=certified.release_id, + ) + artifacts = _required_mapping( + manifest, "artifacts", release_id=certified.release_id + ) + artifact = artifacts.get(key) + if not isinstance(artifact, Mapping): + raise ValueError( + f"Build {certified.build!r} of release {certified.release_id!r} " + f"declares no artifact {key!r}." + ) + name = artifact.get("path") + expected = artifact.get("sha256") + if not isinstance(name, str) or not name or "/" in name: + raise ValueError( + f"Build {certified.build!r} artifact {key!r} has no bare file name." + ) + if not isinstance(expected, str) or not _SHA256_PATTERN.fullmatch(expected): + raise ValueError( + f"Build {certified.build!r} artifact {key!r} has no valid sha256." + ) + path = Path( + hub_download( + repo_id=certified.artifact_repo_id, + filename=f"{folder}/{name}", + repo_type="dataset", + ) + ) + observed = _sha256(path) + if observed != expected.lower(): + raise ValueError( + f"sha256 verification failed for build {certified.build!r} artifact " + f"{key!r}: expected {expected.lower()}; observed {observed}." + ) + return path + + def _dataset_class(spec: DatasetSpec): try: module = importlib.import_module(spec.engine_module) diff --git a/packages/microcosm-data/tests/engine_free/shared/test_loader.py b/packages/microcosm-data/tests/engine_free/shared/test_loader.py index 132afaba9d..b3c50f052f 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_loader.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_loader.py @@ -644,6 +644,113 @@ def test_download_verifies_the_build_folder_dataset( download("uk", 2025) +def test_download_certified_returns_the_current_builds_record( + tmp_path: Path, +) -> None: + hub = _build_folder_hub(tmp_path, manifest=_uk_build_folder_manifest()) + + # The fake build claims policyengine-uk ==2.89.2, which is not installed: + # a comparison reads a build under its own engine, so nothing refuses. + path, certified = loader.download_certified("uk", 2025, hub_download=hub) + + folder = f"releases/{UK_NATIONAL_RELEASE_ID}/{UK_NATIONAL_BUILD}" + assert path.read_bytes() == UK_TAGGED_ARTIFACT + assert certified.release_id == UK_NATIONAL_RELEASE_ID + assert certified.build == UK_NATIONAL_BUILD + assert certified.artifact_path == f"{folder}/microcosm_uk_2024_25.h5" + assert certified.artifact_sha256 == hashlib.sha256(UK_TAGGED_ARTIFACT).hexdigest() + assert certified.model.built_version == "2.89.2" + + +def _declare_build_file( + hub: FakeHubDownload, + tmp_path: Path, + manifest: dict, + *, + key: str, + name: str, + content: bytes, + declared: bytes | None = None, +) -> None: + """Declare ``name`` in the build's manifest and serve ``content`` for it.""" + manifest["artifacts"][key] = { + "kind": "diagnostics", + "path": name, + "repo_id": UK_REPO_ID, + "sha256": hashlib.sha256(content if declared is None else declared).hexdigest(), + "role": key, + } + folder = f"releases/{manifest['release']}/{manifest['build']}" + hub._add( + tmp_path, + UK_REPO_ID, + f"{folder}/release_manifest.json", + None, + json.dumps(manifest).encode(), + ) + hub._add(tmp_path, UK_REPO_ID, f"{folder}/{name}", None, content) + + +def test_download_build_file_reads_a_declared_file_from_the_build_folder( + tmp_path: Path, +) -> None: + manifest = _uk_build_folder_manifest() + hub = _build_folder_hub(tmp_path, manifest=manifest) + _declare_build_file( + hub, + tmp_path, + manifest, + key="release_certification", + name="release_certification.json", + content=b'{"shippable": true}', + ) + _, certified = loader.download_certified("uk", 2025, hub_download=hub) + + path = loader.download_build_file( + certified, "release_certification", hub_download=hub + ) + + folder = f"releases/{UK_NATIONAL_RELEASE_ID}/{UK_NATIONAL_BUILD}" + assert path.read_bytes() == b'{"shippable": true}' + assert hub.calls[-1] == (UK_REPO_ID, f"{folder}/release_certification.json", None) + with pytest.raises(ValueError, match="declares no artifact 'gate_summary'"): + loader.download_build_file(certified, "gate_summary", hub_download=hub) + + +def test_download_build_file_refuses_a_file_whose_digest_differs( + tmp_path: Path, +) -> None: + manifest = _uk_build_folder_manifest() + hub = _build_folder_hub(tmp_path, manifest=manifest) + _declare_build_file( + hub, + tmp_path, + manifest, + key="release_certification", + name="release_certification.json", + content=b'{"shippable": false}', + declared=b'{"shippable": true}', + ) + _, certified = loader.download_certified("uk", 2025, hub_download=hub) + + with pytest.raises(ValueError, match="sha256 verification failed"): + loader.download_build_file(certified, "release_certification", hub_download=hub) + + +def test_download_build_file_needs_a_build_folder_release() -> None: + tagged = loader.CertifiedRelease( + release_id=UK_NATIONAL_RELEASE_ID, + artifact_repo_id=UK_REPO_ID, + artifact_path="microcosm_uk_2024_25.h5", + artifact_revision=UK_NATIONAL_CUT_TAG, + artifact_sha256="0" * 64, + model=loader._PackageCertification("policyengine-uk", ("==2.89.2",), "2.89.2"), + core=loader._PackageCertification("policyengine-core", ("==3.27.1",), "3.27.1"), + ) + with pytest.raises(ValueError, match="keeps the tagged layout"): + loader.download_build_file(tagged, "release_certification") + + def test_uk_2023_follows_latest_json_until_its_repository_has_releases_json( tmp_path: Path, ) -> None: diff --git a/test_support/microcosm_build/uk_cut_reference.py b/test_support/microcosm_build/uk_cut_reference.py new file mode 100644 index 0000000000..3832e114ee --- /dev/null +++ b/test_support/microcosm_build/uk_cut_reference.py @@ -0,0 +1,187 @@ +"""A stand-in for the UK repository's build folders, for cut-reference tests. + +``UKReferenceHub`` answers ``hf_hub_download`` calls from memory for +``policyengine/populace-uk-private``: its ``releases.json`` and the build +folders of the national and dense lines, each with a release manifest, a +dataset and the evidence a cut's fetch reads (the national certification or +the dense gate summary). A missing file raises the Hub's not-found error. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from test_support.microcosm_data.build_folders import build_folder_release_manifest +from test_support.microcosm_data.hub_errors import missing_on_hub + +UK_REPO_ID = "policyengine/populace-uk-private" +UK_NATIONAL_RELEASE = "microcosm-uk-2024-25-national" +UK_DENSE_RELEASE = "microcosm-uk-2024-25-dense" +UK_REGISTERED_KEY_SHA256 = ( + "66bf0e29dbdabc2e3d7a79f21f43ac28d17b592faaf91e67c9bcf9b30c09a6ff" +) + +_LINES = { + UK_NATIONAL_RELEASE: ("national", "microcosm_uk_2024_25"), + UK_DENSE_RELEASE: ("local_area", "microcosm_uk_2024_25_dense"), +} + + +def sha256_of(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def national_certification(dataset: bytes, **overrides: Any) -> dict[str, Any]: + """A certification with the fields a cut's fetch reads, shippable.""" + + certification: dict[str, Any] = { + "kind": "uk_release_certification", + "release_id": UK_NATIONAL_RELEASE, + "shippable": True, + "candidate": {"sha256": sha256_of(dataset)}, + "attestation": {"signing_key_sha256": UK_REGISTERED_KEY_SHA256}, + } + certification.update(overrides) + return certification + + +def dense_gate_summary(**gate_overrides: dict[str, Any]) -> dict[str, Any]: + """A gate summary whose gates all passed, unless overridden.""" + + gates: dict[str, Any] = { + "uk_local_target_fit": {"failures": [], "passed": True, "status": "passed"}, + "uk_local_weight_ratio": {"failures": [], "passed": True, "status": "passed"}, + } + gates.update(gate_overrides) + return { + "schema_version": 1, + "gates": gates, + "diagnostic_gates": {}, + "signed_report": "uk_local_gates.json", + "source": "microcosm_uk_2024_25_dense", + } + + +class UKReferenceHub: + """In-memory ``hf_hub_download`` for the UK repository's build folders.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.calls: list[str] = [] + self._files: dict[str, Path] = {} + self._releases: dict[str, dict[str, Any]] = { + release: {"role": role, "current": None, "builds": {}} + for release, (role, _) in _LINES.items() + } + self._write_record() + + def add_file(self, filename: str, content: bytes) -> None: + local = self.root / "hub" / filename + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(content) + self._files[filename] = local + + def _write_record(self) -> None: + record = { + "schema_version": 1, + "default": UK_NATIONAL_RELEASE, + "releases": self._releases, + } + self.add_file("releases.json", json.dumps(record).encode()) + + def publish( + self, + release: str, + build: str, + *, + dataset: bytes, + evidence: dict[str, Any] | None = None, + current: bool = True, + model_version: str = "2.174.1", + ) -> None: + """Publish ``build`` of ``release`` into its folder; make it current.""" + + _, dataset_key = _LINES[release] + folder = f"releases/{release}/{build}" + evidence_key = ( + "release_certification" + if release == UK_NATIONAL_RELEASE + else "gate_summary" + ) + if evidence is None: + evidence = ( + national_certification(dataset) + if release == UK_NATIONAL_RELEASE + else dense_gate_summary() + ) + evidence_bytes = json.dumps(evidence).encode() + is_national = release == UK_NATIONAL_RELEASE + tagged = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": ( + "national_default" if is_national else "non_default_local_area" + ), + "default_datasets": {"national": dataset_key} if is_national else {}, + "build": { + "built_with_core_package": { + "name": "policyengine-core", + "version": "3.33.3", + }, + "built_with_model_package": { + "name": "policyengine-uk", + "version": model_version, + }, + }, + "compatible_core_packages": [ + {"name": "policyengine-core", "specifier": "==3.33.3"} + ], + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={model_version}"} + ], + "artifacts": { + dataset_key: { + "kind": "microdata", + "path": f"{dataset_key}.h5", + "repo_id": UK_REPO_ID, + "sha256": sha256_of(dataset), + }, + evidence_key: { + "kind": "diagnostics", + "path": f"{evidence_key}.json", + "repo_id": UK_REPO_ID, + "sha256": sha256_of(evidence_bytes), + }, + }, + } + manifest = build_folder_release_manifest(tagged, release=release, build=build) + self.add_file(f"{folder}/release_manifest.json", json.dumps(manifest).encode()) + self.add_file(f"{folder}/{dataset_key}.h5", dataset) + self.add_file(f"{folder}/{evidence_key}.json", evidence_bytes) + entry = self._releases[release] + entry["builds"][build] = { + "published_at": "2026-11-07T03:26:09+00:00", + "withdrawn": False, + } + if current: + entry["current"] = build + self._write_record() + + def __call__( + self, + *, + repo_id: str, + filename: str, + repo_type: str, + revision: str | None = None, + ) -> str: + assert repo_type == "dataset" + assert repo_id == UK_REPO_ID + self.calls.append(filename) + try: + return str(self._files[filename]) + except KeyError as exc: + raise missing_on_hub(repo_id, filename, revision) from exc diff --git a/tools/resolve_uk_reference.py b/tools/resolve_uk_reference.py new file mode 100644 index 0000000000..4738e9a4b6 --- /dev/null +++ b/tools/resolve_uk_reference.py @@ -0,0 +1,94 @@ +"""Resolve the current build a UK cut compares with, and keep its record. + +A UK cut checks and certifies its candidate against the current build of its +own release line, as the Hub's ``releases.json`` names it (microcosm#1095): +the national line against the national, the dense line against the dense. +``tools/run_uk_cut.py`` resolves its reference itself, once, when it fetches +its inputs. This command does the same for runs outside it, such as a dense +build, and checks what a cut would compare with now:: + + uv run --no-sync python tools/resolve_uk_reference.py --release microcosm-uk-2024-25-dense --check + uv run --no-sync python tools/resolve_uk_reference.py --release microcosm-uk-2024-25-dense --output-dir ~/microcosm-uk-work/dense/reference + +``--check`` reads the release record, the build's manifest and its +certification evidence, and downloads no dataset. ``--output-dir`` downloads +the dataset, verifies its SHA-256, copies it there and writes +``reference.json`` beside it; pass both paths to the build's +``--reference-*`` options. It refuses a directory that already holds a +reference, so a run keeps the one it started with. + +Reading ``policyengine/populace-uk-private`` needs ``HF_TOKEN``. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from collections.abc import Sequence +from pathlib import Path + +from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_RELEASES, + check_uk_cut_reference, + resolve_uk_cut_reference, + verify_uk_cut_reference_file, + write_uk_cut_reference, +) + +REFERENCE_RECORD_NAME = "reference.json" + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--release", required=True, choices=sorted(UK_CUT_REFERENCE_RELEASES) + ) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true") + mode.add_argument("--output-dir", type=Path) + return parser.parse_args(argv) + + +def fetch_reference(release: str, output_dir: Path, *, hub_download=None) -> dict: + """Download the current build of ``release`` into ``output_dir``.""" + + record_path = output_dir / REFERENCE_RECORD_NAME + if record_path.exists(): + raise ValueError( + f"{output_dir} already holds a reference ({record_path}); a run keeps " + "the reference it started with. Use a new directory to resolve again." + ) + source, reference = resolve_uk_cut_reference(release, hub_download=hub_download) + output_dir.mkdir(parents=True, exist_ok=True) + destination = output_dir / reference.filename + shutil.copyfile(source, destination) + verify_uk_cut_reference_file(reference, destination) + write_uk_cut_reference(record_path, reference) + return { + "reference": reference.label, + "sha256": reference.sha256, + "dataset": str(destination), + "record": str(record_path), + } + + +def main(argv: Sequence[str] | None = None, *, hub_download=None) -> int: + args = _parse_args(argv) + try: + if args.check: + summary = check_uk_cut_reference(args.release, hub_download=hub_download) + else: + summary = fetch_reference( + args.release, args.output_dir.expanduser(), hub_download=hub_download + ) + except ValueError as exc: + print(f"refused: {exc}", file=sys.stderr) + return 1 + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2d68eaa32b08a1f9e6f59366478d5004e0b66763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:23:23 +0000 Subject: [PATCH 02/18] Compare two scored artifacts in one country-neutral module Both countries score each artifact with microcosm.calibrate's canonical loss, but each wrote its own comparison on top: per target, whose absolute relative error is lower, the counts, the loss gap and the family counts. microcosm.build.head_to_head now holds that comparison once, with no verdict, as microcosm.build.input_mass did for the input-mass gate (#609). The US head-to-head scorer's _comparison_payload calls it and keeps its own payload names, so its scorecard is unchanged: on 300 random fixtures (ties, NaN, signed zeros, empty surfaces) its JSON is byte-identical to main's, and a new test pins the payload. The UK cut's report moves onto the module next (microcosm#1095). Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/head_to_head.py | 253 ++++++++++++++++++ .../engine_free/shared/test_head_to_head.py | 142 ++++++++++ .../us/test_us_release_head_to_head_scorer.py | 49 ++++ tools/score_us_release_head_to_head.py | 92 ++++--- 4 files changed, 499 insertions(+), 37 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/head_to_head.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_head_to_head.py diff --git a/packages/microcosm-build/src/microcosm/build/head_to_head.py b/packages/microcosm-build/src/microcosm/build/head_to_head.py new file mode 100644 index 0000000000..eaf954aad9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/head_to_head.py @@ -0,0 +1,253 @@ +"""Compare a candidate with a reference artifact on one scored target surface. + +Every country scores each artifact on one compiled target surface with +``microcosm.calibrate``'s canonical loss (``score_targets`` / +``relative_error_loss``). This module compares the two scored results, and +nothing else: for each target, whose absolute relative error is lower; how +many targets each side has lower; the loss gap; and the same counts within +each target family. Loading and scoring stay with each country (the US +tool's pools, household slices and by-origin battery; the UK's frame, weight +year and engine measures). + +The comparison decides nothing. It is evidence for the owner's decision to +promote a build, with no threshold applied: the US head-to-head's rule from +its start, and the UK cut's since microcosm#1095 retired its strict rule-1 +verdict against the enhanced FRS. The US tool keeps its own payload's names +("incumbent") for its existing scorecards; the UK receipt uses this module's +("reference"). +""" + +from __future__ import annotations + +from collections.abc import Hashable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +__all__ = [ + "HEAD_TO_HEAD_SCHEMA_VERSION", + "LOWER_CANDIDATE", + "LOWER_EQUAL", + "LOWER_REFERENCE", + "HeadToHead", + "ScoredTarget", + "TargetComparison", + "compare_scored_targets", + "lower_side", + "scored_targets_from_mapping", +] + +HEAD_TO_HEAD_SCHEMA_VERSION = 1 +LOWER_CANDIDATE = "candidate" +LOWER_REFERENCE = "reference" +LOWER_EQUAL = "equal" + + +@dataclass(frozen=True) +class ScoredTarget: + """One target's signed relative error under one artifact.""" + + name: str + relative_error: float + period: Hashable | None = None + family: str | None = None + + +@dataclass(frozen=True) +class TargetComparison: + """One target, scored under both artifacts.""" + + name: str + period: Hashable | None + family: str | None + candidate_relative_error: float + reference_relative_error: float + lower: str + + @property + def candidate_absolute_relative_error(self) -> float: + return abs(self.candidate_relative_error) + + @property + def reference_absolute_relative_error(self) -> float: + return abs(self.reference_relative_error) + + def to_payload(self) -> dict[str, Any]: + return { + "name": self.name, + "period": self.period, + "family": self.family, + "candidate_relative_error": self.candidate_relative_error, + "reference_relative_error": self.reference_relative_error, + "candidate_absolute_relative_error": ( + self.candidate_absolute_relative_error + ), + "reference_absolute_relative_error": ( + self.reference_absolute_relative_error + ), + "candidate_minus_reference_absolute_relative_error": ( + self.candidate_absolute_relative_error + - self.reference_absolute_relative_error + ), + "lower_absolute_relative_error": self.lower, + } + + +def lower_side(candidate_error: float, reference_error: float) -> str: + """Which side's absolute relative error is lower; ``equal`` on a tie. + + Signed errors are compared on magnitude. A comparison involving NaN is + neither lower, so it counts as equal, as both countries' scorers did. + """ + + candidate = abs(float(candidate_error)) + reference = abs(float(reference_error)) + if candidate < reference: + return LOWER_CANDIDATE + if candidate > reference: + return LOWER_REFERENCE + return LOWER_EQUAL + + +def _counts(comparisons: Sequence[TargetComparison]) -> dict[str, int]: + counts = {LOWER_CANDIDATE: 0, LOWER_REFERENCE: 0, LOWER_EQUAL: 0} + for comparison in comparisons: + counts[comparison.lower] += 1 + return { + "candidate_lower_count": counts[LOWER_CANDIDATE], + "reference_lower_count": counts[LOWER_REFERENCE], + "equal_count": counts[LOWER_EQUAL], + } + + +@dataclass(frozen=True) +class HeadToHead: + """Two artifacts compared target by target, with their losses.""" + + candidate_loss: float + reference_loss: float + comparisons: tuple[TargetComparison, ...] + + @property + def counts(self) -> dict[str, int]: + return _counts(self.comparisons) + + def counts_by_family(self) -> dict[str, dict[str, int]]: + """The counts within each family, families sorted; ``None`` is kept.""" + + grouped: dict[str | None, list[TargetComparison]] = {} + for comparison in self.comparisons: + grouped.setdefault(comparison.family, []).append(comparison) + return { + family: _counts(rows) + for family, rows in sorted( + grouped.items(), key=lambda item: (item[0] is None, item[0] or "") + ) + } + + def to_payload(self) -> dict[str, Any]: + """The versioned, country-neutral comparison block, with no verdict.""" + + return { + "schema_version": HEAD_TO_HEAD_SCHEMA_VERSION, + "decision": "owner_decides", + "no_threshold_applied": True, + "loss": { + "candidate": self.candidate_loss, + "reference": self.reference_loss, + "candidate_minus_reference": self.candidate_loss - self.reference_loss, + }, + "targets": { + **self.counts, + "compared": [ + comparison.to_payload() for comparison in self.comparisons + ], + }, + "by_family": self.counts_by_family(), + } + + +def compare_scored_targets( + candidate: Sequence[ScoredTarget], + reference: Sequence[ScoredTarget], + *, + candidate_loss: float, + reference_loss: float, +) -> HeadToHead: + """Compare two scorings of the same target surface, in candidate order. + + Both sequences must name the same targets in the same order (name and + period). A target's family comes from the candidate's scoring, and must + not disagree with the reference's where both give one. + + Raises: + ValueError: If the two scorings cover different targets. + """ + + if len(candidate) != len(reference): + raise ValueError( + "The candidate and reference scorings cover different numbers of " + f"targets ({len(candidate)} and {len(reference)})." + ) + comparisons = [] + for candidate_row, reference_row in zip(candidate, reference, strict=True): + key = (candidate_row.name, candidate_row.period) + reference_key = (reference_row.name, reference_row.period) + if key != reference_key: + raise ValueError( + f"Target comparison contracts differ: {reference_key!r} vs {key!r}." + ) + if ( + candidate_row.family is not None + and reference_row.family is not None + and candidate_row.family != reference_row.family + ): + raise ValueError( + f"Target {key!r} belongs to family {candidate_row.family!r} in the " + f"candidate's scoring but {reference_row.family!r} in the " + "reference's." + ) + comparisons.append( + TargetComparison( + name=candidate_row.name, + period=candidate_row.period, + family=( + candidate_row.family + if candidate_row.family is not None + else reference_row.family + ), + candidate_relative_error=float(candidate_row.relative_error), + reference_relative_error=float(reference_row.relative_error), + lower=lower_side( + candidate_row.relative_error, reference_row.relative_error + ), + ) + ) + return HeadToHead( + candidate_loss=float(candidate_loss), + reference_loss=float(reference_loss), + comparisons=tuple(comparisons), + ) + + +def scored_targets_from_mapping( + errors: Mapping[str, float], + *, + families: Mapping[str, str] | None = None, + order: Sequence[str] | None = None, +) -> list[ScoredTarget]: + """Scored targets from ``{name: signed relative error}``, in ``order``. + + ``order`` defaults to the names sorted; every name in it must be scored. + """ + + names = sorted(errors) if order is None else list(order) + missing = [name for name in names if name not in errors] + if missing: + raise ValueError(f"No score for targets {missing[:10]}.") + families = {} if families is None else families + return [ + ScoredTarget( + name=name, relative_error=float(errors[name]), family=families.get(name) + ) + for name in names + ] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_head_to_head.py b/packages/microcosm-build/tests/engine_free/shared/test_head_to_head.py new file mode 100644 index 0000000000..5dbf548c16 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_head_to_head.py @@ -0,0 +1,142 @@ +"""The country-neutral head-to-head comparison of two scored artifacts.""" + +from __future__ import annotations + +import math + +import pytest + +from microcosm.build.head_to_head import ( + HEAD_TO_HEAD_SCHEMA_VERSION, + LOWER_CANDIDATE, + LOWER_EQUAL, + LOWER_REFERENCE, + ScoredTarget, + compare_scored_targets, + lower_side, + scored_targets_from_mapping, +) + + +@pytest.mark.parametrize( + ("candidate", "reference", "expected"), + [ + (0.1, 0.2, LOWER_CANDIDATE), + (-0.1, 0.2, LOWER_CANDIDATE), + (0.3, -0.2, LOWER_REFERENCE), + (0.2, -0.2, LOWER_EQUAL), + (0.0, -0.0, LOWER_EQUAL), + (math.nan, 0.1, LOWER_EQUAL), + (0.1, math.nan, LOWER_EQUAL), + ], +) +def test_the_lower_side_is_decided_on_magnitude(candidate, reference, expected) -> None: + assert lower_side(candidate, reference) == expected + + +def _scored(errors, families=None): + return [ + ScoredTarget( + name=name, + relative_error=error, + period=2025, + family=None if families is None else families[name], + ) + for name, error in errors.items() + ] + + +def test_the_comparison_counts_targets_and_the_loss_gap() -> None: + families = {"a": "income", "b": "income", "c": "benefits", "d": "benefits"} + head_to_head = compare_scored_targets( + _scored({"a": 0.01, "b": -0.5, "c": 0.2, "d": 0.3}, families), + _scored({"a": -0.02, "b": 0.1, "c": -0.2, "d": 0.1}), + candidate_loss=0.25, + reference_loss=0.75, + ) + + assert head_to_head.counts == { + "candidate_lower_count": 1, + "reference_lower_count": 2, + "equal_count": 1, + } + assert head_to_head.counts_by_family() == { + "benefits": { + "candidate_lower_count": 0, + "reference_lower_count": 1, + "equal_count": 1, + }, + "income": { + "candidate_lower_count": 1, + "reference_lower_count": 1, + "equal_count": 0, + }, + } + payload = head_to_head.to_payload() + assert payload["schema_version"] == HEAD_TO_HEAD_SCHEMA_VERSION + assert payload["decision"] == "owner_decides" + assert payload["no_threshold_applied"] is True + assert payload["loss"] == { + "candidate": 0.25, + "reference": 0.75, + "candidate_minus_reference": -0.5, + } + first = payload["targets"]["compared"][0] + assert first == { + "name": "a", + "period": 2025, + "family": "income", + "candidate_relative_error": 0.01, + "reference_relative_error": -0.02, + "candidate_absolute_relative_error": 0.01, + "reference_absolute_relative_error": 0.02, + "candidate_minus_reference_absolute_relative_error": 0.01 - 0.02, + "lower_absolute_relative_error": LOWER_CANDIDATE, + } + assert "verdict" not in payload + + +def test_targets_without_a_family_are_counted_last() -> None: + head_to_head = compare_scored_targets( + [ScoredTarget("a", 0.1, family="x"), ScoredTarget("b", 0.1)], + [ScoredTarget("a", 0.2), ScoredTarget("b", 0.2)], + candidate_loss=1.0, + reference_loss=1.0, + ) + + assert list(head_to_head.counts_by_family()) == ["x", None] + + +def test_different_surfaces_are_refused() -> None: + with pytest.raises(ValueError, match="different numbers of targets"): + compare_scored_targets( + [ScoredTarget("a", 0.1)], [], candidate_loss=0.0, reference_loss=0.0 + ) + with pytest.raises(ValueError, match="contracts differ"): + compare_scored_targets( + [ScoredTarget("a", 0.1, period=2025)], + [ScoredTarget("a", 0.1, period=2024)], + candidate_loss=0.0, + reference_loss=0.0, + ) + with pytest.raises(ValueError, match="belongs to family"): + compare_scored_targets( + [ScoredTarget("a", 0.1, family="x")], + [ScoredTarget("a", 0.1, family="y")], + candidate_loss=0.0, + reference_loss=0.0, + ) + + +def test_scored_targets_from_a_mapping_follow_the_given_order() -> None: + scored = scored_targets_from_mapping( + {"b": 0.2, "a": 0.1}, families={"a": "x"}, order=["b", "a"] + ) + + assert [(row.name, row.family) for row in scored] == [("b", None), ("a", "x")] + assert [row.name for row in scored_targets_from_mapping({"b": 0.2, "a": 0.1})] == [ + "a", + "b", + ] + with pytest.raises(ValueError, match="No score for targets"): + scored_targets_from_mapping({"a": 0.1}, order=["a", "z"]) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py b/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py index b9af195866..4c2129a43b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py @@ -1325,3 +1325,52 @@ def corrupt_materialize(slice_frame, specs, **kwargs): assert type(caught.value) is expected_type errors.append((type(caught.value), str(caught.value).encode("utf-8"))) assert errors[0] == errors[1] + + +def test_the_comparison_payload_keeps_the_us_scorecard_names() -> None: + """The shared comparison (microcosm.build.head_to_head) leaves this + payload, and so the written scorecard, exactly as it was.""" + + def side(errors, loss, status): + return { + "fiscal": { + "targets": [ + { + "name": name, + "period": 2024, + "absolute_relative_error": abs(error), + } + for name, error in errors + ], + "weighted_loss": loss, + }, + "terminal_battery": {"status": status}, + } + + module = _load_head_to_head_module() + payload = module._comparison_payload( + side([("a", 0.2), ("b", 0.1), ("c", 0.3)], 0.5, "computed_finished_h5"), + side([("a", 0.1), ("b", 0.1), ("c", -0.4)], 0.25, "computed_finished_h5"), + ) + + assert payload["decision"] == "owner_decides_flip" + assert payload["fiscal_weighted_loss"] == { + "incumbent": 0.5, + "candidate": 0.25, + "candidate_minus_incumbent": -0.25, + } + per_target = payload["per_target_absolute_relative_error"] + assert ( + per_target["candidate_lower_count"], + per_target["equal_count"], + per_target["incumbent_lower_count"], + ) == (1, 1, 1) + assert per_target["targets"][2] == { + "name": "c", + "period": 2024, + "incumbent_absolute_relative_error": 0.3, + "candidate_absolute_relative_error": 0.4, + "candidate_minus_incumbent_absolute_relative_error": 0.4 - 0.3, + "lower_absolute_relative_error": "incumbent", + } + assert payload["terminal_battery"]["head_to_head_comparable"] is True diff --git a/tools/score_us_release_head_to_head.py b/tools/score_us_release_head_to_head.py index 098540048e..eda715e2d7 100644 --- a/tools/score_us_release_head_to_head.py +++ b/tools/score_us_release_head_to_head.py @@ -72,6 +72,13 @@ import pandas as pd import score_us_fiscal_targets as fiscal_scorer +from microcosm.build.head_to_head import ( + LOWER_CANDIDATE, + LOWER_EQUAL, + LOWER_REFERENCE, + ScoredTarget, + compare_scored_targets, +) from microcosm.build.us_runtime.congressional_district_vintage import ( CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, @@ -2094,44 +2101,55 @@ def _comparison_payload( raise TypeError("Artifact target payloads must be lists.") if len(incumbent_rows) != len(candidate_rows): raise ValueError("Fiscal target payload lengths differ after contract checks.") - target_comparisons: list[dict[str, object]] = [] - candidate_lower = 0 - equal = 0 - incumbent_lower = 0 - for incumbent_row, candidate_row in zip( - incumbent_rows, - candidate_rows, - strict=True, - ): - key = (incumbent_row["name"], incumbent_row["period"]) - candidate_key = (candidate_row["name"], candidate_row["period"]) - if candidate_key != key: - raise ValueError( - f"Target comparison contracts differ: {key!r} vs {candidate_key!r}." + # The country-neutral comparison (microcosm.build.head_to_head); this + # payload keeps the US scorecard's own names, so its bytes are unchanged. + head_to_head = compare_scored_targets( + [ + ScoredTarget( + name=row["name"], + period=row["period"], + relative_error=float(row["absolute_relative_error"]), ) - incumbent_error = float(incumbent_row["absolute_relative_error"]) - candidate_error = float(candidate_row["absolute_relative_error"]) - if candidate_error < incumbent_error: - candidate_lower += 1 - lower = "candidate" - elif candidate_error > incumbent_error: - incumbent_lower += 1 - lower = "incumbent" - else: - equal += 1 - lower = "equal" - target_comparisons.append( - { - "name": key[0], - "period": key[1], - "incumbent_absolute_relative_error": incumbent_error, - "candidate_absolute_relative_error": candidate_error, - "candidate_minus_incumbent_absolute_relative_error": ( - candidate_error - incumbent_error - ), - "lower_absolute_relative_error": lower, - } - ) + for row in candidate_rows + ], + [ + ScoredTarget( + name=row["name"], + period=row["period"], + relative_error=float(row["absolute_relative_error"]), + ) + for row in incumbent_rows + ], + candidate_loss=float(candidate_fiscal["weighted_loss"]), + reference_loss=float(incumbent_fiscal["weighted_loss"]), + ) + lower_names = { + LOWER_CANDIDATE: "candidate", + LOWER_REFERENCE: "incumbent", + LOWER_EQUAL: "equal", + } + target_comparisons: list[dict[str, object]] = [ + { + "name": comparison.name, + "period": comparison.period, + "incumbent_absolute_relative_error": ( + comparison.reference_absolute_relative_error + ), + "candidate_absolute_relative_error": ( + comparison.candidate_absolute_relative_error + ), + "candidate_minus_incumbent_absolute_relative_error": ( + comparison.candidate_absolute_relative_error + - comparison.reference_absolute_relative_error + ), + "lower_absolute_relative_error": lower_names[comparison.lower], + } + for comparison in head_to_head.comparisons + ] + counts = head_to_head.counts + candidate_lower = counts["candidate_lower_count"] + equal = counts["equal_count"] + incumbent_lower = counts["reference_lower_count"] incumbent_loss = float(incumbent_fiscal["weighted_loss"]) candidate_loss = float(candidate_fiscal["weighted_loss"]) incumbent_battery = incumbent["terminal_battery"] From 90c9279bd8be410ed534c0c446d5fac59fe42269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:56:01 +0000 Subject: [PATCH 03/18] Compare UK national cuts with the current national build, as a report A cut's candidate is now compared with the current build of the national line, which the Hub's releases.json names, instead of the enhanced FRS (microcosm#1095), and the comparison is evidence with no verdict, as the US head-to-head is. - run_uk_cut resolves the reference once per cut, in a new reference step before the spine, into /reference/ with its record. The national, score and certify steps read that record; none resolves again, so a promotion mid-cut cannot split it. --re-resolve-reference sets it aside. status says when the current build has moved; stage 1 compares with the reference by default. The incumbent_efrs input and its gates.json pin selector are gone. - candidate_score builds on microcosm.build.head_to_head. Its receipt, score_vs_reference.json, schema 2, names the reference and carries the losses, per-target comparisons and family counts, with no verdict. Measures the reference cannot carry are pruned under reference_unresolvable_measures.json, scoped per reference build; the enhanced FRS's eight entries retire with it. - The national role, the build CLI (--reference-h5, --reference-record) and the score command follow; the dense role refuses the national role's reference flags. - The certifier requires a schema-2 receipt for this candidate and the cut's own reference, and the certification (schema 2) names that reference. The release contract moves to schema 2 and binds score_vs_reference.json, which the assembler now publishes. Only new builds are validated against the contract, at the spec they are assembled under, so schema 1 keeps no path. The input-mass, export-surface and input-coverage gates still read the enhanced-FRS references; they move to the current build next. Co-Authored-By: Claude Opus 5.5 --- .../microcosm/build/uk/country_package.json | 2 +- .../uk/incumbent_unresolvable_measures.json | 62 --- .../uk/reference_unresolvable_measures.json | 5 + .../build/uk_runtime/candidate_score.py | 425 +++++++++--------- .../build/uk_runtime/cut_reference.py | 43 ++ .../build/uk_runtime/full_build_cli.py | 28 +- .../build/uk_runtime/national_role.py | 146 +++--- .../build/uk_runtime/release_certification.py | 134 +++--- .../microcosm/build/uk_runtime/rowwise_cli.py | 6 +- .../engine_free/shared/test_country_spec.py | 4 +- .../tests/engine_free/uk/test_run_uk_cut.py | 264 +++++++++-- .../uk/test_score_uk_national_candidate.py | 234 +++++++--- .../engine_free/uk/test_uk_candidate_score.py | 305 ++++++++----- .../uk/test_uk_certify_release_cut_command.py | 16 +- .../engine_free/uk/test_uk_full_build_cli.py | 2 +- .../uk/test_uk_local_incumbent_extractor.py | 2 +- .../uk/test_uk_release_assembler.py | 4 +- .../uk/test_uk_release_certification.py | 115 +++-- .../uk/test_uk_rowwise_national_role.py | 266 ++++++----- .../tests/uk_certification_fixtures.py | 82 +++- .../src/microcosm/data/contract.py | 43 +- .../tests/engine_free/shared/test_contract.py | 46 +- tools/assemble_uk_release_dir.py | 9 +- tools/certify_uk_release_cut.py | 17 +- tools/extract_uk_local_incumbent_surface.py | 7 +- tools/resolve_uk_reference.py | 25 +- tools/run_uk_cut.py | 216 +++++++-- tools/score_uk_national_candidate.py | 105 +++-- tools/uk_cut_inputs.json | 14 +- 29 files changed, 1738 insertions(+), 889 deletions(-) delete mode 100644 packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/reference_unresolvable_measures.json diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 14ac473a8e..7bc487061b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -348,7 +348,7 @@ "schema_id": "legacy_json" }, { - "path": "incumbent_unresolvable_measures.json", + "path": "reference_unresolvable_measures.json", "kind": "legacy_json", "schema_id": "legacy_json" }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json b/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json deleted file mode 100644 index cdf4583bcd..0000000000 --- a/packages/microcosm-build/src/microcosm/build/uk/incumbent_unresolvable_measures.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "schema_version": 2, - "description": "Measures the incumbent comparison artifact (the enhanced FRS) is known not to carry, keyed entity.variable. A target bound through a listed measure may be pruned from BOTH arms of the rule-1 evaluation (microcosm#578 rule 1 on the common surface, microcosm#823); an unlisted unresolvable measure refuses the evaluation rather than pruning. Every entry is a complete approval receipt in the reviewed-exclusion schema, in force from approved_on through expires_on.", - "exclusions": { - "benunit.uc_calibration_administrative_family_type": { - "reason": "Administrative-basis UC family type (microcosm#832/#835/#937): derived from the relationship-to-head reconstruction (microcosm#791) the enhanced FRS never carried; 93 dwp_universal_credit rows bind through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - }, - "benunit.uc_calibration_child_count": { - "reason": "Administrative-basis UC dependent-child count (microcosm#832/#835/#937): the same relationship-to-head derivation (microcosm#791) the enhanced FRS never carried.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - }, - "benunit.uc_calibration_has_child_under_one": { - "reason": "Administrative-basis UC child-under-one flag (microcosm#832/#835/#937): the same relationship-to-head derivation (microcosm#791) the enhanced FRS never carried.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - }, - "benunit.uc_tcl_affected_benunit_proxy": { - "reason": "Two-child-limit affected-benunit proxy (microcosm#840): a counterfactual the enhanced FRS carries no inputs for; 15 dwp_two_child_limit rows bind through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - }, - "household.bus_in_london_boardings": { - "reason": "NTS bus-travel stage (microcosm#930): the enhanced FRS carries no bus travel, so it cannot materialize household boardings on London buses; the dft.bus_journeys London row (microcosm#1207, bound since microcosm#1286) binds through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578); the bus journeys family was bound after the incumbent pin (microcosm#1286).", - "approved_on": "2026-10-19", - "expires_on": "2027-01-19" - }, - "household.ons_household_type": { - "reason": "ONS census household type (microcosm#887): the enhanced FRS carries no household-type classification; 10 ons_household_composition rows bind through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - }, - "household.other_local_bus_boardings": { - "reason": "NTS bus-travel stage (microcosm#930): the enhanced FRS carries no bus travel, so it cannot materialize household boardings on local buses outside London; the 8 dft.bus_journeys rows for the English regions outside London (microcosm#1207, bound since microcosm#1286) bind through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578); the bus journeys family was bound after the incumbent pin (microcosm#1286).", - "approved_on": "2026-10-19", - "expires_on": "2027-01-19" - }, - "person.capital_gains_asset_type": { - "reason": "CGT asset type (microcosm#939, the E8 spine stage): the enhanced FRS carries gains amounts only; 2 hmrc_cgt rows bind through it.", - "approved_by": "juaristi22", - "adjudication": "microcosm#823 (rule-1 evaluation on the common surface; microcosm#578)", - "approved_on": "2026-09-22", - "expires_on": "2026-12-22" - } - } -} diff --git a/packages/microcosm-build/src/microcosm/build/uk/reference_unresolvable_measures.json b/packages/microcosm-build/src/microcosm/build/uk/reference_unresolvable_measures.json new file mode 100644 index 0000000000..cffd86b261 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/reference_unresolvable_measures.json @@ -0,0 +1,5 @@ +{ + "schema_version": 3, + "description": "Measures a reference build is known not to carry, scoped per reference (microcosm#1095): each key is a reference's @, and each entry under it, keyed entity.variable, lets a target bound through that measure be pruned from BOTH arms of the head-to-head comparison on the common surface (microcosm#823). An unlisted unresolvable measure refuses the comparison rather than pruning. Entries under a build that is no longer the cut's reference are dormant. Every entry is a complete approval receipt in the reviewed-exclusion schema, in force from approved_on through expires_on.", + "references": {} +} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py index 432e0a219b..9b5b314dad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/candidate_score.py @@ -1,19 +1,25 @@ -"""Score a UK national candidate against the incumbent on one frozen register. +"""Compare a UK national candidate with the current build on one frozen register. -The #578 rule-1 score core, shared by the scorer command -(``tools/score_uk_national_candidate.py``), the rowwise driver's national -role (which evaluates every build at its end) and the release-cut certifier -(which refuses a receipt whose verdict is not ``passed``). +The head-to-head behind every UK national cut (microcosm#1095), shared by +the scorer command (``tools/score_uk_national_candidate.py``), the rowwise +driver's national role (which evaluates every build at its end) and the +release-cut certifier (which requires a receipt for this candidate and this +reference). The reference is the current build of the national line, as the +cut recorded it when it fetched its inputs (:mod:`.cut_reference`). Both artifacts are materialized through the route the calibration seam -uses. A target whose measure the *candidate* cannot materialize refuses: -that is a defect. A target whose measure the *incumbent* cannot materialize -is structural — the incumbent artifact never carried the input (the -admin-basis UC family measures, the CGT asset type, the ONS household type -on the enhanced FRS) — so :func:`evaluate_uk_candidate_against_incumbent` -prunes it from both arms, scores the common surface with band edges from -the full register (#803), and reports every pruned measure loudly in the -receipt rather than quietly shrinking the surface or refusing the score. +uses, and compared target by target through the country-neutral +:mod:`microcosm.build.head_to_head`. The comparison decides nothing: it is +evidence for the owner's decision to promote the candidate, with no +threshold, as the US head-to-head is. Until microcosm#1095 the reference was +the enhanced FRS and the receipt carried a strict rule-1 verdict. + +A target whose measure the *candidate* cannot materialize refuses: that is a +defect. A target whose measure the *reference* cannot materialize (a build +from before an input existed) is pruned from both arms under a signed entry +of the reviewed reference-unresolvable register for that reference, scored +on the common surface with band edges from the full register (#803), and +reported in the receipt. """ from __future__ import annotations @@ -26,6 +32,10 @@ from pathlib import Path from typing import Any +from microcosm.build.head_to_head import ( + compare_scored_targets, + scored_targets_from_mapping, +) from microcosm.build.target_materialization import MeasureResolutionError from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame from microcosm.build.uk_runtime.national_frame import ( @@ -39,7 +49,7 @@ from microcosm.build.uk_runtime.weighted_integrity import ( UKReviewedExclusion, exclusion_evaluation_date, - load_uk_reviewed_exclusion_register, + load_uk_reference_scoped_exclusion_register, ) from microcosm.calibrate import TargetRegistry, score_targets @@ -69,17 +79,17 @@ def _holdout_loss(basis: str) -> float | None: def score_uk_national_candidate( *, candidate_h5: str | Path, - incumbent_h5: str | Path, + reference_h5: str | Path, candidate_sha256: str, - incumbent_sha256: str, + reference_sha256: str, target_registry: TargetRegistry, calibration_year: int, + reference_label: str, measure_resolver_factory: Callable[[Path, Any], Any] | None = None, candidate_label: str | None = None, - incumbent_label: str = "enhanced_frs_2024_25", band_edge_registry: TargetRegistry | None = None, ) -> dict[str, Any]: - """Return the #578 rule-1 score block on a shared target registry. + """Return the head-to-head score block on a shared target registry. Both artifacts are verified against their declared digests before a byte is read, and both sides are materialized through the same route the @@ -93,7 +103,7 @@ def score_uk_national_candidate( if candidate_label is None: candidate_label = candidate_path.stem candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) - incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) + reference_pin = _verify_artifact("reference", reference_h5, reference_sha256) candidate_frame, candidate_resolution = _scored_frame( candidate_h5, target_registry, @@ -101,43 +111,49 @@ def score_uk_national_candidate( measure_resolver_factory, band_edge_registry=band_edge_registry, ) - incumbent_frame, incumbent_resolution = _scored_frame( - incumbent_h5, + reference_frame, reference_resolution = _scored_frame( + reference_h5, target_registry, calibration_year, measure_resolver_factory, band_edge_registry=band_edge_registry, - incumbent=True, + reference=True, ) candidate = score_targets( candidate_frame, target_registry.to_target_set(), target_loss_cap=UK_SCORE_LOSS_CAP, ) - incumbent = score_targets( - incumbent_frame, + reference = score_targets( + reference_frame, target_registry.to_target_set(), target_loss_cap=UK_SCORE_LOSS_CAP, ) candidate_errors = _relative_errors(candidate) - incumbent_errors = _relative_errors(incumbent) + reference_errors = _relative_errors(reference) names = sorted(candidate_errors) - missing = sorted(set(names) ^ set(incumbent_errors)) + missing = sorted(set(names) ^ set(reference_errors)) if missing: raise RuntimeError( - "candidate and incumbent scores produced different target rows: " + "candidate and reference scores produced different target rows: " f"{missing[:10]}." ) - wins = _target_wins(candidate_errors, incumbent_errors) + families = { + spec.to_target().row_name: spec.family for spec in target_registry.specs + } + head_to_head = compare_scored_targets( + scored_targets_from_mapping(candidate_errors, families=families, order=names), + scored_targets_from_mapping(reference_errors, families=families, order=names), + candidate_loss=float(candidate.final_loss), + reference_loss=float(reference.final_loss), + ) return { "candidate_train_loss": float(candidate.final_loss), "candidate_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), "candidate_full_loss": float(candidate.final_loss), - "incumbent_train_loss": float(incumbent.final_loss), - "incumbent_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), - "incumbent_full_loss": float(incumbent.final_loss), - "candidate_target_wins": wins["candidate"], - "incumbent_target_wins": wins["incumbent"], + "reference_train_loss": float(reference.final_loss), + "reference_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), + "reference_full_loss": float(reference.final_loss), "holdout_basis": UK_SCORE_HOLDOUT_BASIS, "loss": { "objective": "relative_error_loss", @@ -151,11 +167,11 @@ def score_uk_national_candidate( }, "artifacts": { "candidate": {"label": candidate_label, **candidate_pin}, - "incumbent": {"label": incumbent_label, **incumbent_pin}, + "reference": {"label": reference_label, **reference_pin}, }, "measure_resolution": { "candidate": candidate_resolution, - "incumbent": incumbent_resolution, + "reference": reference_resolution, }, # Each file's own year and the factor the engine multiplies its # household weights by to reach the scored year. Scores taken before @@ -163,33 +179,19 @@ def score_uk_national_candidate( "weight_year": { "scored_year": int(calibration_year), "candidate": _weight_year(candidate_frame, calibration_year), - "incumbent": _weight_year(incumbent_frame, calibration_year), + "reference": _weight_year(reference_frame, calibration_year), }, - "target_drift": _target_drift( - target_registry, candidate_errors, incumbent_errors - ), + "head_to_head": head_to_head.to_payload(), "signed_asymmetries": [ { - "id": "incumbent_own_registry", + "id": "reference_own_registry", "description": ( - "The incumbent was calibrated to its own registry; both " - "artifacts are rescored here on the supplied frozen register." - ), - }, - { - "id": "national_direct_vs_collapsed_local", - "description": ( - "Candidate national weights are a direct national solve; " - "incumbent national weights are the published national " - "surface being compared on the same frozen register." + "The reference was calibrated to the register of its own " + "build; both artifacts are rescored here on the " + "candidate's frozen register." ), }, ], - "target_wins_by_family": _target_wins_by_family( - target_registry, - candidate_errors, - incumbent_errors, - ), } @@ -210,44 +212,6 @@ def _relative_errors(result) -> dict[str, float]: } -def _target_wins( - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> dict[str, int]: - candidate = 0 - incumbent = 0 - for name, candidate_error in candidate_errors.items(): - incumbent_error = incumbent_errors[name] - if abs(candidate_error) < abs(incumbent_error): - candidate += 1 - elif abs(incumbent_error) < abs(candidate_error): - incumbent += 1 - return {"candidate": candidate, "incumbent": incumbent} - - -def _target_wins_by_family( - registry: TargetRegistry, - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> dict[str, dict[str, int]]: - by_row_name = {spec.to_target().row_name: spec.family for spec in registry.specs} - result: dict[str, dict[str, int]] = {} - for name, family in by_row_name.items(): - bucket = result.setdefault( - family, - {"candidate_target_wins": 0, "incumbent_target_wins": 0, "ties": 0}, - ) - candidate_error = abs(candidate_errors[name]) - incumbent_error = abs(incumbent_errors[name]) - if candidate_error < incumbent_error: - bucket["candidate_target_wins"] += 1 - elif incumbent_error < candidate_error: - bucket["incumbent_target_wins"] += 1 - else: - bucket["ties"] += 1 - return result - - def _load_registry(path: str | Path) -> TargetRegistry: """Load the frozen register through its own validating loader. @@ -280,9 +244,16 @@ def _verify_artifact(role: str, path: str | Path, expected: str) -> dict[str, ob } -#: Rule 1 scores each file's best estimate of the year's income against the -#: same annual targets; an incumbent built before the annual income -#: descriptors existed has none, so its annual measures are its current ones. +#: The comparison scores each file's best estimate of the year's income +#: against the same annual targets; a reference built before the annual +#: income descriptors existed has none, so its annual measures are its +#: current ones. +UK_REFERENCE_ANNUAL_INCOME_FALLBACK = ( + "reference without annual income descriptors: its annual measures are its " + "current values (a share of 1 and no other spells)" +) +#: The same fallback for the enhanced FRS, which the dense line's opt-in +#: local diagnostic still reads (tools/extract_uk_local_incumbent_surface.py). UK_INCUMBENT_ANNUAL_INCOME_FALLBACK = ( "incumbent without annual income descriptors: its annual measures are its " "current values (a share of 1 and no other spells)" @@ -296,7 +267,7 @@ def _scored_frame( factory: Callable[[Path, Any], Any] | None, *, band_edge_registry: TargetRegistry | None = None, - incumbent: bool = False, + reference: bool = False, ) -> tuple[Any, Any]: frame, _provenance = load_uk_national_frame(h5_path) # policyengine-uk projects a file's household weights from the file's own @@ -305,8 +276,8 @@ def _scored_frame( frame = uk_engine_year_frame(frame, simulation_year=calibration_year) resolver = None if factory is None else factory(Path(h5_path), frame) fallback = getattr(resolver, "use_current_income_for_annual_measures", None) - if incumbent and fallback is not None: - fallback(reason=UK_INCUMBENT_ANNUAL_INCOME_FALLBACK) + if reference and fallback is not None: + fallback(reason=UK_REFERENCE_ANNUAL_INCOME_FALLBACK) return prepare_uk_target_frame( frame, registry, @@ -337,63 +308,33 @@ def build(h5_path: Path, frame: Any): return build -def _target_drift( - registry: TargetRegistry, - candidate_errors: dict[str, float], - incumbent_errors: dict[str, float], -) -> list[dict[str, object]]: - """Per-target relative errors, the auditable half of the score block.""" - - families = {spec.to_target().row_name: spec.family for spec in registry.specs} - rows = [] - for name in sorted(candidate_errors): - candidate = candidate_errors[name] - incumbent = incumbent_errors[name] - # Relative errors are signed; the win is decided on magnitude, the - # same rule the aggregate counts use. - rows.append( - { - "target": name, - "family": families.get(name), - "candidate_relative_error": candidate, - "incumbent_relative_error": incumbent, - "winner": ( - "candidate" - if abs(candidate) < abs(incumbent) - else "incumbent" - if abs(incumbent) < abs(candidate) - else "tie" - ), - } - ) - return rows - - -#: The rule the verdict decides: microcosm#578 rule 1, on the surface both -#: artifacts can materialize. +#: What the comparison is (schema 2, microcosm#1095; schema 1 was the strict +#: rule-1 verdict against the enhanced FRS). UK_EVALUATION_RULE = ( - "microcosm#578 rule 1: the candidate beats the incumbent on the frozen " - "comparison register, both artifacts rescored on the surface both can " - "materialize; rows the incumbent cannot materialize are pruned from both " - "arms under the reviewed incumbent-unresolvable register and listed." + "microcosm#1095: the candidate and the current build of its release line " + "(the reference), both rescored on the candidate's frozen register on the " + "surface both can materialize, compared target by target with no " + "threshold; rows the reference cannot materialize are pruned from both " + "arms under the reviewed reference-unresolvable register and listed." ) -UK_EVALUATION_SCHEMA_VERSION = 1 -UK_EVALUATION_VERDICT_PASSED = "passed" -UK_EVALUATION_VERDICT_FAILED = "failed" -INCUMBENT_UNRESOLVABLE_NOTE = ( +UK_EVALUATION_SCHEMA_VERSION = 2 +REFERENCE_UNRESOLVABLE_NOTE = ( "Rows pruned from BOTH arms because their measures cannot be computed on " - "the incumbent (inputs the incumbent artifact does not carry), each under " - "a signed entry of the reviewed incumbent-unresolvable register. The " - "candidate's fit on these rows is measured by the calibration diagnostics " - "and the terminal gates instead." + "the reference (inputs the reference build does not carry), each under a " + "signed entry of the reviewed reference-unresolvable register for this " + "reference. The candidate's fit on these rows is measured by the " + "calibration diagnostics and the terminal gates instead." ) -#: The reviewed register of measures the incumbent is known not to carry -#: (``microcosm.build.uk`` resource, reviewed-exclusion schema 2, keyed -#: ``entity.variable``). Pruning a target from both arms is a doctrine -#: decision, not a scorer's inference: only a measure with a signed, in-force -#: entry may be pruned, and an unlisted one refuses the evaluation. -UK_INCUMBENT_UNRESOLVABLE_REGISTER = "incumbent_unresolvable_measures.json" +#: The reviewed register of measures a reference build is known not to carry +#: (``microcosm.build.uk`` resource, the per-reference schema 3, keyed by the +#: reference's ``@`` and then ``entity.variable``). Pruning a +#: target from both arms is a doctrine decision, not a scorer's inference: +#: only a measure with a signed, in-force entry for the cut's own reference +#: may be pruned, and an unlisted one refuses the evaluation. Entries for +#: another build are dormant: they go unused once a promotion moves the +#: reference past them. +UK_REFERENCE_UNRESOLVABLE_REGISTER = "reference_unresolvable_measures.json" #: A materialization skip's reason names the missing column as #: ``'entity.variable'`` (or ``'variable'``); the prune loop matches that @@ -401,17 +342,28 @@ def _target_drift( _SKIP_COLUMN = re.compile(r"^'(?:([a-z_0-9]+)\.)?([A-Za-z_0-9]+)'$") -def load_uk_incumbent_unresolvable_measures( +def load_uk_reference_unresolvable_register( source: str | Path | None = None, -) -> dict[str, UKReviewedExclusion]: - """The reviewed incumbent-unresolvable register (packaged unless overridden).""" +) -> dict[str, Mapping[str, UKReviewedExclusion]]: + """Every reference's reviewed unresolvable measures (packaged by default).""" - return load_uk_reviewed_exclusion_register( - source, resource=UK_INCUMBENT_UNRESOLVABLE_REGISTER + return load_uk_reference_scoped_exclusion_register( + source, resource=UK_REFERENCE_UNRESOLVABLE_REGISTER ) -def uk_incumbent_unresolvable_register_digest( +def load_uk_reference_unresolvable_measures( + *, + reference_label: str, + source: str | Path | None = None, +) -> dict[str, UKReviewedExclusion]: + """The reviewed unresolvable measures for one reference, empty if none.""" + + register = load_uk_reference_unresolvable_register(source) + return dict(register.get(reference_label, {})) + + +def uk_reference_unresolvable_register_digest( source: str | Path | None = None, ) -> dict[str, str]: """Name and digest of the register a receipt was evaluated under.""" @@ -421,10 +373,10 @@ def uk_incumbent_unresolvable_register_digest( raw = ( importlib_resources.files("microcosm.build.uk") - .joinpath(UK_INCUMBENT_UNRESOLVABLE_REGISTER) + .joinpath(UK_REFERENCE_UNRESOLVABLE_REGISTER) .read_bytes() ) - name = UK_INCUMBENT_UNRESOLVABLE_REGISTER + name = UK_REFERENCE_UNRESOLVABLE_REGISTER else: raw = Path(source).read_bytes() name = str(source) @@ -449,20 +401,22 @@ def _require_reviewed_unresolvable( variable: str, *, evaluated_on: date, + reference_label: str, ) -> UKReviewedExclusion: key = f"{entity}.{variable}" record = reviewed.get(key) if record is None: raise MeasureResolutionError( - f"{key} is unresolvable on the incumbent but is not on the reviewed " - f"incumbent-unresolvable register ({UK_INCUMBENT_UNRESOLVABLE_REGISTER}); " - "a target may be pruned from both arms only under a signed entry, " - "so the evaluation refuses rather than pruning.", + f"{key} is unresolvable on the reference {reference_label} but has no " + "entry for it on the reviewed reference-unresolvable register " + f"({UK_REFERENCE_UNRESOLVABLE_REGISTER}); a target may be pruned " + "from both arms only under a signed entry, so the evaluation " + "refuses rather than pruning.", receipt={}, ) if record.premature(evaluated_on): raise MeasureResolutionError( - f"the reviewed incumbent-unresolvable entry for {key} is not yet in " + f"the reviewed reference-unresolvable entry for {key} is not yet in " f"force (takes force {record.approved_on}, evaluated on " f"{evaluated_on.isoformat()}); correct the receipt's approved_on or " "wait for it.", @@ -470,7 +424,7 @@ def _require_reviewed_unresolvable( ) if record.expired(evaluated_on): raise MeasureResolutionError( - f"the reviewed incumbent-unresolvable entry for {key} expired " + f"the reviewed reference-unresolvable entry for {key} expired " f"{record.expires_on} (evaluated on {evaluated_on.isoformat()}); renew " "the adjudication or remove the entry.", receipt={}, @@ -488,7 +442,7 @@ def _require_reviewed_unresolvable( @dataclass(frozen=True) class PrunedTarget: - """One target dropped from both arms because the incumbent lacks its measure.""" + """One target dropped from both arms because the reference lacks its measure.""" name: str family: str @@ -523,8 +477,8 @@ def _packaged_contract_targets() -> Mapping[str, Any]: return _uk_contract_targets() -def prune_incumbent_unresolvable( - incumbent_h5: str | Path, +def prune_reference_unresolvable( + reference_h5: str | Path, registry: TargetRegistry, calibration_year: int, measure_resolver_factory: Callable[[Path, Any], Any] | None, @@ -534,14 +488,16 @@ def prune_incumbent_unresolvable( max_rounds: int = 10, reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, evaluated_on: date | None = None, + reference_label: str, ) -> tuple[TargetRegistry, dict[str, PrunedTarget]]: - """Drop every target whose measure the incumbent cannot materialize. + """Drop every target whose measure the reference cannot materialize. - Probes the incumbent's resolution on the scoring surface; on each + Probes the reference's resolution on the scoring surface; on each :class:`MeasureResolutionError` naming a measure, requires a signed, - in-force entry for that measure on the reviewed incumbent-unresolvable - register (``reviewed_measures``, the packaged register by default, - evaluated on ``evaluated_on``, today by default), then drops the targets + in-force entry for that measure on the reviewed reference-unresolvable + register for ``reference_label`` (``reviewed_measures``, the packaged + register's entries for it by default, evaluated on ``evaluated_on``, + today by default), then drops the targets the resolution receipt's skip rows name for it, else every target whose contract binding gates or values through it, and records each drop with the failing measure and the entry's adjudication. An unlisted measure @@ -552,7 +508,7 @@ def prune_incumbent_unresolvable( if measure_resolver_factory is None: return registry, {} reviewed = ( - load_uk_incumbent_unresolvable_measures() + load_uk_reference_unresolvable_measures(reference_label=reference_label) if reviewed_measures is None else reviewed_measures ) @@ -565,12 +521,12 @@ def prune_incumbent_unresolvable( for _round in range(max_rounds): try: _scored_frame( - incumbent_h5, + reference_h5, surface, calibration_year, measure_resolver_factory, band_edge_registry=edges, - incumbent=True, + reference=True, ) except MeasureResolutionError as error: failing = _failing_measure(error) @@ -578,7 +534,11 @@ def prune_incumbent_unresolvable( raise entity, variable = failing record = _require_reviewed_unresolvable( - reviewed, entity, variable, evaluated_on=now + reviewed, + entity, + variable, + evaluated_on=now, + reference_label=reference_label, ) receipt = getattr(error, "receipt", None) or {} skips = receipt.get("skips", []) if isinstance(receipt, Mapping) else [] @@ -606,7 +566,7 @@ def prune_incumbent_unresolvable( ] if not dropped: raise MeasureResolutionError( - f"{entity}.{variable} is unresolvable on the incumbent but " + f"{entity}.{variable} is unresolvable on the reference but " "neither the resolution receipt nor any contract binding " "names a target for it; refusing to prune blindly.", receipt=receipt if isinstance(receipt, Mapping) else {}, @@ -626,13 +586,13 @@ def prune_incumbent_unresolvable( ) if not surface.specs: raise RuntimeError( - "every target is unresolvable on the incumbent; nothing " + "every target is unresolvable on the reference; nothing " "remains to score." ) from error continue return surface, pruned raise RuntimeError( - f"the incumbent's measure resolution still fails after {max_rounds} " + f"the reference's measure resolution still fails after {max_rounds} " "pruning rounds." ) @@ -642,6 +602,7 @@ def pruned_block( *, n_scored: int, n_surface: int, + reference_label: str, reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, register: Mapping[str, str] | None = None, ) -> dict[str, Any]: @@ -666,63 +627,61 @@ def pruned_block( "measures": measures, "families": dict(sorted(families.items())), # The candidate materialized every target of the full surface before - # the incumbent's gaps were pruned; only the incumbent is pruned for. + # the reference's gaps were pruned; only the reference is pruned for. "candidate_validated_on_full_surface": True, "reviewed_register": { **( - {"resource": UK_INCUMBENT_UNRESOLVABLE_REGISTER, "sha256": None} + {"resource": UK_REFERENCE_UNRESOLVABLE_REGISTER, "sha256": None} if register is None else dict(register) ), + "reference": reference_label, "entries_used": entries_used, }, - "note": INCUMBENT_UNRESOLVABLE_NOTE, + "note": REFERENCE_UNRESOLVABLE_NOTE, } def evaluation_block(score: Mapping[str, Any]) -> dict[str, Any]: - """The machine-checkable verdict the certifier reads from a receipt.""" + """The comparison's summary the certifier reads from a receipt: no verdict.""" - candidate_loss = float(score["candidate_full_loss"]) - incumbent_loss = float(score["incumbent_full_loss"]) - pruned = score.get("incumbent_unresolvable_pruned") or {} + pruned = score.get("reference_unresolvable_pruned") or {} n_scored = int(pruned.get("n_scored", score["register"]["n_specs"])) n_pruned = int(pruned.get("n_pruned", 0)) n_surface = int(pruned.get("n_surface", n_scored + n_pruned)) - rule_1_passed = candidate_loss < incumbent_loss + targets = score["head_to_head"]["targets"] return { "schema_version": UK_EVALUATION_SCHEMA_VERSION, "rule": UK_EVALUATION_RULE, + "reference": score["artifacts"]["reference"]["label"], "scored_surface": { "n_scored": n_scored, "n_pruned": n_pruned, "n_surface": n_surface, }, - "rule_1": { - "passed": rule_1_passed, - "candidate_full_loss": candidate_loss, - "incumbent_full_loss": incumbent_loss, - "candidate_target_wins": int(score["candidate_target_wins"]), - "incumbent_target_wins": int(score["incumbent_target_wins"]), + "losses": { + "candidate_full_loss": float(score["candidate_full_loss"]), + "reference_full_loss": float(score["reference_full_loss"]), }, - "verdict": ( - UK_EVALUATION_VERDICT_PASSED - if rule_1_passed - else UK_EVALUATION_VERDICT_FAILED - ), + "targets": { + "candidate_lower_count": int(targets["candidate_lower_count"]), + "reference_lower_count": int(targets["reference_lower_count"]), + "equal_count": int(targets["equal_count"]), + }, + "no_threshold_applied": True, } def pruned_warning(score: Mapping[str, Any]) -> str | None: """The loud warning for a receipt that pruned rows, or None.""" - pruned = score.get("incumbent_unresolvable_pruned") or {} + pruned = score.get("reference_unresolvable_pruned") or {} if not pruned.get("n_pruned"): return None lines = [ f"warning: {pruned['n_pruned']} of {pruned['n_surface']} targets cannot be " - "materialized on the incumbent and were pruned from BOTH arms; the " - f"score stands on {pruned['n_scored']} common targets:" + "materialized on the reference and were pruned from BOTH arms; the " + f"comparison stands on {pruned['n_scored']} common targets:" ] targets = pruned.get("pruned_targets", {}) for measure in pruned.get("measures", []): @@ -736,55 +695,68 @@ def pruned_warning(score: Mapping[str, Any]) -> str | None: ) lines.append( " the candidate's fit on these rows is measured by the calibration " - "diagnostics and the terminal gates, not by this score." + "diagnostics and the terminal gates, not by this comparison." ) return "\n".join(lines) -def evaluate_uk_candidate_against_incumbent( +def evaluate_uk_candidate_against_reference( *, candidate_h5: str | Path, - incumbent_h5: str | Path, + reference_h5: str | Path, candidate_sha256: str, - incumbent_sha256: str, + reference_sha256: str, target_registry: TargetRegistry, calibration_year: int, + reference_label: str, measure_resolver_factory: Callable[[Path, Any], Any] | None = None, candidate_label: str | None = None, - incumbent_label: str = "enhanced_frs_2024_25", band_edge_registry: TargetRegistry | None = None, - prune_incumbent_unresolvable_measures: bool = True, + prune_reference_unresolvable_measures: bool = True, contract_targets: Mapping[str, Any] | None = None, reviewed_measures: Mapping[str, UKReviewedExclusion] | None = None, evaluated_on: date | None = None, + reference_record: Mapping[str, Any] | None = None, ) -> dict[str, Any]: - """Score on the common resolvable surface and decide the rule-1 verdict. + """Compare on the common resolvable surface and summarize, with no verdict. - Returns the score block with ``incumbent_unresolvable_pruned`` (every + Returns the score block with ``reference_unresolvable_pruned`` (every row dropped, its family, the absent measure and the register entry it - was pruned under) and ``evaluation`` (rule, counts, rule-1 losses and - wins, ``verdict``). Pruning stands on the reviewed incumbent-unresolvable - register (the packaged one unless ``reviewed_measures`` is supplied); - with pruning off the strict scorer's refusal stands. + was pruned under) and ``evaluation`` (rule, reference, counts, losses). + Pruning stands on the reviewed reference-unresolvable register's entries + for ``reference_label`` (the packaged register unless ``reviewed_measures`` + is supplied); with pruning off the strict scorer's refusal stands. + ``reference_record`` (the cut's :class:`~.cut_reference.UKCutReference` + record) is kept in the receipt as ``reference``; it must name the scored + file. """ + if reference_record is not None and ( + reference_record.get("sha256") != reference_sha256 + or f"{reference_record.get('release')}@{reference_record.get('build')}" + != reference_label + ): + raise ValueError( + "The reference record names another build than the scored reference " + f"({reference_label}, {reference_sha256})." + ) surface = target_registry pruned: dict[str, PrunedTarget] = {} register_digest = ( - uk_incumbent_unresolvable_register_digest() + uk_reference_unresolvable_register_digest() if reviewed_measures is None else None ) reviewed = ( - load_uk_incumbent_unresolvable_measures() + load_uk_reference_unresolvable_measures(reference_label=reference_label) if reviewed_measures is None else reviewed_measures ) - if prune_incumbent_unresolvable_measures: + if prune_reference_unresolvable_measures: # The candidate is validated on the FULL surface before any pruning: # a measure the candidate cannot materialize is a defect and refuses - # whether or not the incumbent shares the gap (review finding A1), - # so pruning only ever removes rows the incumbent cannot express. + # whether or not the reference shares the gap (review finding A1), + # so pruning only ever removes rows the reference cannot express. _scored_frame( candidate_h5, target_registry, @@ -794,8 +766,8 @@ def evaluate_uk_candidate_against_incumbent( target_registry if band_edge_registry is None else band_edge_registry ), ) - surface, pruned = prune_incumbent_unresolvable( - incumbent_h5, + surface, pruned = prune_reference_unresolvable( + reference_h5, target_registry, calibration_year, measure_resolver_factory, @@ -803,6 +775,7 @@ def evaluate_uk_candidate_against_incumbent( contract_targets=contract_targets, reviewed_measures=reviewed, evaluated_on=evaluated_on, + reference_label=reference_label, ) # A pruned surface must never redraw its own band edges (#803): the # edges come from the full register when the caller supplied none. @@ -811,22 +784,24 @@ def evaluate_uk_candidate_against_incumbent( edges = target_registry score = score_uk_national_candidate( candidate_h5=candidate_h5, - incumbent_h5=incumbent_h5, + reference_h5=reference_h5, candidate_sha256=candidate_sha256, - incumbent_sha256=incumbent_sha256, + reference_sha256=reference_sha256, target_registry=surface, calibration_year=calibration_year, measure_resolver_factory=measure_resolver_factory, candidate_label=candidate_label, - incumbent_label=incumbent_label, + reference_label=reference_label, band_edge_registry=edges, ) - score["incumbent_unresolvable_pruned"] = pruned_block( + score["reference_unresolvable_pruned"] = pruned_block( pruned, n_scored=len(surface.specs), n_surface=len(target_registry.specs), + reference_label=reference_label, reviewed_measures=reviewed, register=register_digest, ) + score["reference"] = None if reference_record is None else dict(reference_record) score["evaluation"] = evaluation_block(score) return score diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py index ce1a803ecc..bb41a27794 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py @@ -24,6 +24,7 @@ import hashlib import json +import shutil from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -40,7 +41,9 @@ "UK_CUT_REFERENCE_RELEASES", "UK_CUT_REFERENCE_SCHEMA_VERSION", "UKCutReference", + "UK_CUT_REFERENCE_RECORD_NAME", "check_uk_cut_reference", + "fetch_uk_cut_reference", "read_uk_cut_reference", "resolve_uk_cut_reference", "verify_uk_cut_reference_file", @@ -49,6 +52,8 @@ UK_CUT_REFERENCE_KIND = "uk_cut_reference" UK_CUT_REFERENCE_SCHEMA_VERSION = 1 +#: The record's file name beside the reference's dataset in a cut. +UK_CUT_REFERENCE_RECORD_NAME = "reference.json" #: Each line's registry variant and the build evidence its fetch reads. UK_CUT_REFERENCE_RELEASES: Mapping[str, tuple[str, str]] = { @@ -420,3 +425,41 @@ def verify_uk_cut_reference_file(reference: UKCutReference, path: str | Path) -> f"{reference.label} has {reference.sha256}." ) return source + + +def fetch_uk_cut_reference( + release: str, + output_dir: str | Path, + *, + hub_download: _HubDownload | None = None, + now: Callable[[], datetime] | None = None, +) -> UKCutReference: + """Resolve the current build of ``release`` into ``output_dir``, once. + + Copies the verified dataset there and writes its record beside it + (:data:`UK_CUT_REFERENCE_RECORD_NAME`). Refuses a directory that already + holds a record, so a run keeps the reference it started with. + """ + + directory = Path(output_dir) + record_path = directory / UK_CUT_REFERENCE_RECORD_NAME + if record_path.exists(): + raise ValueError( + f"{directory} already holds a reference ({record_path}); a run keeps " + "the reference it started with. Use a new directory to resolve again." + ) + source, reference = resolve_uk_cut_reference( + release, hub_download=hub_download, now=now + ) + directory.mkdir(parents=True, exist_ok=True) + destination = directory / reference.filename + partial = destination.with_name(destination.name + ".partial") + shutil.copyfile(source, partial) + try: + verify_uk_cut_reference_file(reference, partial) + except ValueError: + partial.unlink(missing_ok=True) + raise + partial.replace(destination) + write_uk_cut_reference(record_path, reference) + return reference diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 8c2c5ae682..361b711041 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -825,18 +825,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="National role only: allow a Ledger artifact whose feed commit is not the committed Chronicle pin.", ) parser.add_argument( - "--incumbent-h5", + "--reference-h5", type=Path, - help="National role only: the incumbent dataset the finished candidate is evaluated against.", - ) - parser.add_argument( - "--incumbent-sha256", - help="National role only: the incumbent's SHA-256, verified before it is read.", + help=( + "National role only: the current national build the finished " + "candidate is compared with (microcosm#1095)." + ), ) parser.add_argument( - "--incumbent-label", - default="enhanced_frs_2024_25", - help="National role only: the incumbent's label in the score receipt.", + "--reference-record", + type=Path, + help=( + "National role only: the cut's record of that build " + "(tools/resolve_uk_reference.py), checked against the file." + ), ) parser.add_argument( "--logbook-prev-row-digest", @@ -3194,7 +3196,7 @@ def _execute_national_build( "chronicle_provenance": dict(registry_document["chronicle_provenance"]), } record["inputs"] = inputs - record["incumbent"] = national_role.incumbent_arguments(args) + record["reference"] = national_role.reference_arguments(args) # 3. The solve and the seam battery, evaluated in the graph. stage( telemetry, @@ -3528,7 +3530,7 @@ def _close_national_attempt( """Stage the published bundle, evaluate, close the telemetry, spool the row. The seam's order: the manifest and the staged bundle's hand-over first, - then the incumbent evaluation, which waits for no upload and runs before + then the comparison with the reference, which waits for no upload and runs before the telemetry completes, then the build record and the manifest rewritten with the delivery summary and the sums refreshed. The build record is rewritten after the hand-over, so the bundle the emitter holds keeps the @@ -3556,9 +3558,9 @@ def _close_national_attempt( ) staged_dataset = staged.delivery append_phase(state, STAGED_DATASET_PHASES[staged_dataset["status"]]) - evaluation = national_role.evaluate_against_incumbent( + evaluation = national_role.evaluate_against_reference( args, - incumbent=record.get("incumbent"), + reference=record.get("reference"), inputs=record["inputs"], output_paths=paths, telemetry=telemetry, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py index e52e1b2656..2386df83c2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_role.py @@ -5,8 +5,8 @@ national path lives in :mod:`.full_build_cli`). This module holds what the role adds around the graph: the dry-run plan, the receipted doctrine overrides, the schema-4 national manifest over the seam-shaped evidence, the -optional end-of-build evaluation against the incumbent (microcosm#578 rule 1 -on the common surface), the pinned-input and output-path checks and the +optional end-of-build comparison with the reference (the current national +build, microcosm#1095), the pinned-input and output-path checks and the local sums. These functions were moved from ``tools/build_uk_rowwise_candidate.py`` (microcosm#901 phase 4) and consume only the shared rowwise command surface (:mod:`.rowwise_cli`, @@ -57,11 +57,11 @@ from microcosm.calibrate import TargetRegistry __all__ = [ - "evaluate_against_incumbent", - "incumbent_arguments", + "evaluate_against_reference", "national_doctrine_overrides", "national_dry_run", "national_manifest", + "reference_arguments", "require_bound_input", "require_signing_key", ] @@ -146,6 +146,15 @@ def require_signing_key() -> None: ) from error +def _dry_run_reference(args: argparse.Namespace) -> dict[str, Any] | None: + """The reference a dry run would compare with, without its full record.""" + + reference = reference_arguments(args) + if reference is None: + return None + return {key: value for key, value in reference.items() if key != "record"} + + def national_dry_run( args: argparse.Namespace, *, @@ -203,7 +212,7 @@ def national_dry_run( ), ), "engine": "not_run", - "incumbent": incumbent_arguments(args), + "reference": _dry_run_reference(args), "releasable": False, } if operation_inventory is not None: @@ -418,7 +427,7 @@ def verify_requested_pin( def _load_candidate_evaluator(importer=importlib.import_module): - """The common-surface scorer (microcosm#967), loaded when an incumbent is given. + """The head-to-head scorer, loaded when a reference is given. Lazy so the driver imports without it; a build asked to evaluate refuses up front, before any solve, when the scorer is not in the tree. @@ -428,33 +437,45 @@ def _load_candidate_evaluator(importer=importlib.import_module): return importer("microcosm.build.uk_runtime.candidate_score") except ImportError as error: raise ValueError( - "--incumbent-h5 needs the common-surface scorer (microcosm#967): " + "--reference-h5 needs the head-to-head scorer: " "microcosm.build.uk_runtime.candidate_score is not in this tree." ) from error -def incumbent_arguments(args: argparse.Namespace) -> dict[str, Any] | None: - """The national role's optional incumbent, pinned and verified up front.""" +def reference_arguments(args: argparse.Namespace) -> dict[str, Any] | None: + """The national role's optional reference, verified against its record. + + The reference is the current build the cut recorded when it fetched it + (:mod:`.cut_reference`); the file must be the one the record names. + """ + + from microcosm.build.uk_runtime.cut_reference import ( + read_uk_cut_reference, + verify_uk_cut_reference_file, + ) - if args.incumbent_h5 is None and args.incumbent_sha256 is None: + if args.reference_h5 is None and args.reference_record is None: return None - if args.incumbent_h5 is None or args.incumbent_sha256 is None: + if args.reference_h5 is None or args.reference_record is None: raise ValueError( - "--incumbent-h5 and --incumbent-sha256 must be given together." + "--reference-h5 and --reference-record must be given together." ) _load_candidate_evaluator() - incumbent_h5 = require_file(args.incumbent_h5, label="--incumbent-h5") - info = artifact_info(incumbent_h5) - verify_requested_pin("--incumbent-h5", info, requested=args.incumbent_sha256) + reference_h5 = require_file(args.reference_h5, label="--reference-h5") + record = read_uk_cut_reference( + require_file(args.reference_record, label="--reference-record") + ) + verify_uk_cut_reference_file(record, reference_h5) return { - "path": str(incumbent_h5), - "sha256": info["sha256"], - "bytes": info["bytes"], - "label": str(args.incumbent_label), + "path": str(reference_h5), + "sha256": record.sha256, + "bytes": record.size_bytes, + "label": record.label, + "record": record.to_record(), } -_incumbent_arguments = incumbent_arguments +_reference_arguments = reference_arguments def list_sha256sums_entry(out_dir: Path, name: str) -> None: @@ -476,127 +497,146 @@ def list_sha256sums_entry(out_dir: Path, name: str) -> None: #: Receipt blocks that are record arrays (lists of mappings): the reviewed -#: telemetry artifact policy refuses them, and the verdict, the pruned block -#: and the aggregates carry everything a reviewer reads. The full receipt stays +#: telemetry artifact policy refuses them, and the summary, the pruned block +#: and the counts carry everything a reviewer reads. The full receipt stays #: beside the outputs and in the staged bundle. _SCORE_RECEIPT_TELEMETRY_EXCLUDED = ( - "target_drift", "signed_asymmetries", "measure_resolution", ) def _score_receipt_telemetry_summary(score: Mapping[str, Any]) -> dict[str, Any]: - return { + summary = { key: value for key, value in score.items() if key not in _SCORE_RECEIPT_TELEMETRY_EXCLUDED } + head_to_head = summary.get("head_to_head") + if isinstance(head_to_head, Mapping): + targets = head_to_head.get("targets") + summary["head_to_head"] = { + **head_to_head, + "targets": { + key: value + for key, value in (targets or {}).items() + if key != "compared" + }, + } + return summary -def evaluate_against_incumbent( +def evaluate_against_reference( args: argparse.Namespace, *, - incumbent: Mapping[str, Any] | None, + reference: Mapping[str, Any] | None, inputs: Mapping[str, Any], output_paths: Mapping[str, Path], telemetry: StagingRunBundleWriterV2 | None, calibration_year: int, out_dir: Path, ) -> dict[str, Any]: - """Score the finished candidate against the incumbent; never fail the build. + """Compare the finished candidate with the reference; never fail the build. Runs after the staged bundle is on the Hub and before the telemetry completes, so the receipt rides the run as a reviewed artifact. Rows the - incumbent cannot materialize are pruned from both arms and warned about - by name; the verdict (microcosm#578 rule 1 on the common surface) is - what the release-cut certifier requires to be ``passed``. An error is - recorded and warned, never raised: staging and the exit code stand. + reference cannot materialize are pruned from both arms and warned about + by name. The comparison decides nothing (microcosm#1095); the release-cut + certifier requires its receipt to name this candidate and this reference. + An error is recorded and warned, never raised: staging and the exit code + stand. """ - if incumbent is None: + if reference is None: return { "status": "not_requested", "note": ( - "no --incumbent-h5: the rule-1 score receipt the release-cut " - "certifier needs was not produced; score the candidate with " + "no --reference-h5: the score receipt the release-cut certifier " + "needs was not produced; score the candidate with " "tools/score_uk_national_candidate.py before certification." ), } stage( telemetry, - "incumbent_evaluation", + "reference_evaluation", "started", - incumbent_sha256=incumbent["sha256"], + reference=reference["label"], + reference_sha256=reference["sha256"], ) candidate = artifact_info(output_paths["dataset"]) try: module = _load_candidate_evaluator() - score = module.evaluate_uk_candidate_against_incumbent( + score = module.evaluate_uk_candidate_against_reference( candidate_h5=output_paths["dataset"], - incumbent_h5=Path(incumbent["path"]), + reference_h5=Path(reference["path"]), candidate_sha256=candidate["sha256"], - incumbent_sha256=incumbent["sha256"], + reference_sha256=reference["sha256"], target_registry=inputs["national_registry"], calibration_year=calibration_year, + reference_label=reference["label"], measure_resolver_factory=module.uk_default_measure_resolver_factory( out_dir, calibration_year ), candidate_label=output_paths["dataset"].stem, - incumbent_label=incumbent["label"], band_edge_registry=inputs["band_edge_registry"], + reference_record=reference["record"], ) atomic_write_json(output_paths["score_receipt"], score) except Exception as error: # noqa: BLE001 - the evaluation never fails a finished build message = f"{type(error).__name__}: {error}"[:600] print( - f"warning: the incumbent evaluation failed ({message}); the build's " - "evidence and staging are unaffected, and the candidate cannot be " - "certified until it is re-scored with " + f"warning: the comparison with the reference failed ({message}); the " + "build's evidence and staging are unaffected, and the candidate " + "cannot be certified until it is re-scored with " "tools/score_uk_national_candidate.py.", file=sys.stderr, flush=True, ) - stage(telemetry, "incumbent_evaluation", "failed", error=message) + stage(telemetry, "reference_evaluation", "failed", error=message) return { "status": "error", "error": message, - "incumbent": dict(incumbent), + "reference": { + key: value for key, value in reference.items() if key != "record" + }, "receipt": None, } warning = module.pruned_warning(score) if warning is not None: print(warning, file=sys.stderr, flush=True) evaluation = score["evaluation"] - pruned = score["incumbent_unresolvable_pruned"] + pruned = score["reference_unresolvable_pruned"] add_staging_artifact( telemetry, - "score_vs_incumbent", + "score_vs_reference", _score_receipt_telemetry_summary(score), artifact_kind="aggregate_diagnostics", classification="aggregate", ) stage( telemetry, - "incumbent_evaluation", + "reference_evaluation", "completed", - verdict=evaluation["verdict"], + reference=evaluation["reference"], n_scored=int(evaluation["scored_surface"]["n_scored"]), n_pruned=int(evaluation["scored_surface"]["n_pruned"]), ) return { "status": "completed", - "verdict": evaluation["verdict"], - "rule_1": dict(evaluation["rule_1"]), + "reference": evaluation["reference"], + "losses": dict(evaluation["losses"]), + "targets": dict(evaluation["targets"]), "scored_surface": dict(evaluation["scored_surface"]), "pruned_measures": list(pruned["measures"]), "pruned_families": dict(pruned["families"]), "receipt": artifact_info(output_paths["score_receipt"]), - "incumbent": dict(incumbent), + "reference_input": { + key: value for key, value in reference.items() if key != "record" + }, } -_evaluate_against_incumbent = evaluate_against_incumbent +_evaluate_against_reference = evaluate_against_reference def validate_output_paths( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 9915dbad2e..3181ec6ad1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -79,7 +79,10 @@ "uk_release_parity_evidence", ] -UK_RELEASE_CERTIFICATION_SCHEMA_VERSION = 1 +#: Schema 2 (microcosm#1095) names the reference the cut was compared with, +#: the current national build, and reads a head-to-head receipt with no +#: verdict; schema 1 compared with the enhanced FRS under a strict rule. +UK_RELEASE_CERTIFICATION_SCHEMA_VERSION = 2 UK_RELEASE_CERTIFICATION_KIND = "uk_release_certification" UK_RELEASE_CUT_POSTURE = "release_cut" UK_CERTIFICATION_EXCLUDED_GATE_IDS = frozenset(UK_LOCAL_GATE_SCOPE) | frozenset( @@ -414,6 +417,7 @@ def compose_uk_release_certification( spine_sidecar: Mapping[str, Any], build_record: Mapping[str, Any], score_receipt_path: Path, + reference_record: Mapping[str, Any], exclusions_evaluated_on: date, certification_path: Path, reviewed_unresolvable_measures: Mapping[str, Any] | None = None, @@ -425,9 +429,15 @@ def compose_uk_release_certification( ``UK_SHARED_GATE_IDS``, full phase coverage, verified signatures, the committed spec's scoped digests, green release-blocking verdicts, and a closed identity join from the spine report through the sidecar, the - build record, the diagnostics digest, and the candidate bytes. + build record, the diagnostics digest, and the candidate bytes, and a + head-to-head receipt that measured this candidate against the cut's own + reference (``reference_record``, the current national build the cut + recorded when it fetched it). """ + from microcosm.build.uk_runtime.cut_reference import UKCutReference + + reference = UKCutReference.from_record(reference_record) parts_raw = { "spine": _load_part(spine_report_path), "calibration_seam": _load_part(seam_report_path), @@ -491,6 +501,7 @@ def compose_uk_release_certification( _verify_score_receipt( score_receipt, candidate_sha256=candidate_sha256, + reference=reference, reviewed_unresolvable_measures=reviewed_unresolvable_measures, evaluated_on=exclusions_evaluated_on, ) @@ -511,6 +522,15 @@ def compose_uk_release_certification( # The spine whose stage receipts the release cut measured: bound to # the parent the calibration recorded, never an operator's choice. "parent_spine": {"sha256": spine_sha256}, + # What the cut was compared with: the current national build when + # the cut fetched its inputs (microcosm#1095). + "reference": { + "release": reference.release, + "build": reference.build, + "filename": reference.filename, + "sha256": reference.sha256, + "built_with_model_package": dict(reference.built_with_model_package), + }, "parts": part_summaries, "spec": { "gates_manifest_sha256": full_digests["gates_manifest_sha256"], @@ -867,14 +887,15 @@ def _verify_identity_join( ) -#: The verdict a certifiable score receipt must carry (candidate_score). -UK_SCORE_RECEIPT_VERDICT_PASSED = "passed" +#: The head-to-head receipt a certification reads (candidate_score schema 2). +UK_SCORE_RECEIPT_SCHEMA_VERSION = 2 def _verify_score_receipt( receipt: Mapping[str, Any], *, candidate_sha256: str, + reference: Any, reviewed_unresolvable_measures: Mapping[str, Any] | None = None, evaluated_on: date | None = None, ) -> None: @@ -888,20 +909,42 @@ def _verify_score_receipt( raise UKReleaseCertificationError( "the score receipt's artifacts.candidate.sha256 is " f"{scored!r}, not the candidate under certification " - f"({candidate_sha256!r}); the rule-1 score must be measured on " + f"({candidate_sha256!r}); the comparison must be measured on " "this candidate's bytes." ) - # Rule 1 (microcosm#578) is decided by the scorer on the surface both - # artifacts can materialize and signed into the receipt; the certifier - # refuses anything short of a passed verdict, so publication never runs - # on an unpassed evaluation. Pruned rows are reported, never a failure, - # but the surface must close over them. + # The comparison (microcosm#1095) decides nothing; the certifier requires + # that it was made, on this candidate, against the cut's own reference, + # and that its surface closes over the rows it pruned. evaluation = receipt.get("evaluation") - if not isinstance(evaluation, Mapping): + if ( + not isinstance(evaluation, Mapping) + or evaluation.get("schema_version") != UK_SCORE_RECEIPT_SCHEMA_VERSION + ): raise UKReleaseCertificationError( - "the score receipt carries no evaluation block; re-score the " - "candidate with tools/score_uk_national_candidate.py so the " - "rule-1 verdict is signed run evidence." + "the score receipt carries no schema-2 evaluation block; re-score " + "the candidate against the cut's reference with " + "tools/score_uk_national_candidate.py." + ) + scored_reference = ( + artifacts.get("reference", {}).get("sha256") + if isinstance(artifacts, Mapping) + else None + ) + recorded = receipt.get("reference") + named = ( + (recorded.get("release"), recorded.get("build"), recorded.get("sha256")) + if isinstance(recorded, Mapping) + else None + ) + if scored_reference != reference.sha256 or named != ( + reference.release, + reference.build, + reference.sha256, + ): + raise UKReleaseCertificationError( + "the score receipt compared the candidate with another reference " + f"than the cut's ({reference.label}, {reference.sha256}); re-score " + "the candidate against the reference the cut recorded." ) surface = evaluation.get("scored_surface") counts = ( @@ -919,26 +962,20 @@ def _verify_score_receipt( "the score receipt's scored surface does not close: n_scored + " "n_pruned must equal n_surface with at least one scored target." ) - verdict = evaluation.get("verdict") - if verdict != UK_SCORE_RECEIPT_VERDICT_PASSED: - raise UKReleaseCertificationError( - f"the score receipt's evaluation verdict is {verdict!r}, not " - f"{UK_SCORE_RECEIPT_VERDICT_PASSED!r}: rule 1 (microcosm#578) is " - "not met on the common surface, so the cut cannot be certified." - ) - # Every pruned measure must stand on the committed reviewed register: - # the scorer's inference is re-checked here against the register of - # record, so a receipt scored under another register cannot certify. - pruned = receipt.get("incumbent_unresolvable_pruned") + # Every pruned measure must stand on the committed reviewed register, + # under the cut's own reference: the scorer's inference is re-checked + # here against the register of record, so a receipt scored under another + # register, or another reference's entries, cannot certify. + pruned = receipt.get("reference_unresolvable_pruned") measures = pruned.get("measures", []) if isinstance(pruned, Mapping) else [] if measures: from microcosm.build.uk_runtime.candidate_score import ( - load_uk_incumbent_unresolvable_measures, - uk_incumbent_unresolvable_register_digest, + load_uk_reference_unresolvable_measures, + uk_reference_unresolvable_register_digest, ) reviewed = ( - load_uk_incumbent_unresolvable_measures() + load_uk_reference_unresolvable_measures(reference_label=reference.label) if reviewed_unresolvable_measures is None else reviewed_unresolvable_measures ) @@ -946,24 +983,21 @@ def _verify_score_receipt( if unlisted: raise UKReleaseCertificationError( "the score receipt pruned targets through measure(s) not on the " - f"reviewed incumbent-unresolvable register: {unlisted}; pruning " - "from both arms stands only on a signed entry, so the cut " - "cannot be certified on this receipt." + "reviewed reference-unresolvable register for " + f"{reference.label}: {unlisted}; pruning from both arms stands " + "only on a signed entry, so the cut cannot be certified on this " + "receipt." ) - # The receipt names the register it was scored under; against the - # committed register of record that digest must be the committed - # one, so a receipt scored under another register cannot certify. if reviewed_unresolvable_measures is None: - named = ( - pruned.get("reviewed_register", {}).get("sha256") - if isinstance(pruned.get("reviewed_register"), Mapping) - else None + register = pruned.get("reviewed_register") + named_digest = ( + register.get("sha256") if isinstance(register, Mapping) else None ) - committed = uk_incumbent_unresolvable_register_digest()["sha256"] - if named != committed: + committed = uk_reference_unresolvable_register_digest()["sha256"] + if named_digest != committed: raise UKReleaseCertificationError( - "the score receipt was scored under incumbent-unresolvable " - f"register {named!r}, not the committed register " + "the score receipt was scored under reference-unresolvable " + f"register {named_digest!r}, not the committed register " f"({committed}); re-score the candidate on the register of " "record." ) @@ -984,7 +1018,7 @@ def _verify_score_receipt( if out_of_window: raise UKReleaseCertificationError( "the score receipt pruned targets through reviewed " - "incumbent-unresolvable entries outside their window on " + "reference-unresolvable entries outside their window on " f"{evaluated_on.isoformat()}: {out_of_window}; renew the " "adjudication or re-score the candidate." ) @@ -993,21 +1027,13 @@ def _verify_score_receipt( def _score_receipt_summary(receipt: Mapping[str, Any]) -> dict[str, Any]: evaluation = receipt["evaluation"] surface = evaluation["scored_surface"] - rule_1 = evaluation.get("rule_1") - losses = ( - { - "candidate_full_loss": rule_1.get("candidate_full_loss"), - "incumbent_full_loss": rule_1.get("incumbent_full_loss"), - } - if isinstance(rule_1, Mapping) - else {} - ) return { - "verdict": evaluation["verdict"], + "reference": evaluation["reference"], "n_scored": surface["n_scored"], "n_pruned": surface["n_pruned"], "n_surface": surface["n_surface"], - **losses, + **dict(evaluation["losses"]), + **dict(evaluation["targets"]), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 5676178464..3d41a2664d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -125,7 +125,7 @@ BUILD_RECORD_FILENAME = "build_record.json" NATIONAL_REGISTRY_FILENAME = "national_target_registry.json" NATIONAL_CONTRACT_REGISTRY_FILENAME = "national_contract_registry.json" -SCORE_RECEIPT_FILENAME = "score_vs_incumbent.json" +SCORE_RECEIPT_FILENAME = "score_vs_reference.json" DENSE_REFERENCE_DIAGNOSTICS_FILENAME = "dense_reference_diagnostics.csv" DATASET_SIZE_SELECTION_FILENAME = "dataset_size_selection.csv" @@ -1249,8 +1249,8 @@ def refuse_national_role_arguments( refused.append("--target-loss-cap") if args.allow_unpinned_feed: refused.append("--allow-unpinned-feed") - if args.incumbent_h5 is not None or args.incumbent_sha256 is not None: - refused.append("--incumbent-h5/--incumbent-sha256") + if args.reference_h5 is not None or args.reference_record is not None: + refused.append("--reference-h5/--reference-record") if args.target_weight_rule not in posture.allowed_target_weight_rules: refused.append(f"--target-weight-rule {args.target_weight_rule}") if refused: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index dccee82499..ee3a33e071 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -1127,7 +1127,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "take_up_contract.json", "target_reference_signed_exclusions.json", "input_mass_reviewed_exclusions.json", - "incumbent_unresolvable_measures.json", + "reference_unresolvable_measures.json", "spine_swap_signed_differences.json", "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", @@ -1305,7 +1305,7 @@ def test_uk_package_loads(self) -> None: "take_up_contract.json", "target_reference_signed_exclusions.json", "input_mass_reviewed_exclusions.json", - "incumbent_unresolvable_measures.json", + "reference_unresolvable_measures.json", "spine_swap_signed_differences.json", "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py index 291ff28c80..18192048d9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py @@ -45,7 +45,6 @@ def test_every_declared_input_resolves_to_a_committed_pin(): "hmrc_personal_incomes_ods", "chronicle_uk", "input_mass_reference", - "incumbent_efrs", } for source in sources.values(): assert source.files @@ -80,10 +79,6 @@ def test_the_runner_pins_are_the_build_pins(): assert sources["frs_2024_25"].revision == frs["huggingface_revision"] assert sources["frs_2024_25"].archive_sha256 == frs["zip_sha256"] assert sources["frs_2024_25"].filename == frs["zip_filename"] - identity = cut._gate_reference_identity(cut.UK_DIR) - incumbent = sources["incumbent_efrs"] - assert incumbent.revision == identity["revision"] - assert incumbent.files[0].sha256 == identity["sha256"] stage_pins = cut.committed_source_pins() for key in ( "frs_2024_25", @@ -190,6 +185,40 @@ def _paths(tmp_path: Path) -> cut.CutPaths: return cut.CutPaths(tmp_path, "test", "a" * 40) +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" + + +def _reference(paths: cut.CutPaths, *, build: str = REFERENCE_BUILD, data=b"ref"): + """Put a reference and its record in the cut, as the reference step does.""" + + from microcosm.build.uk_runtime.cut_reference import ( + UKCutReference, + write_uk_cut_reference, + ) + + reference = UKCutReference( + release=cut.RELEASE_ID, + build=build, + repo_id=cut.PUBLISH_REPO, + path_in_repo=f"releases/{cut.RELEASE_ID}/{build}/microcosm_uk_2024_25.h5", + filename="microcosm_uk_2024_25.h5", + sha256=_sha(data), + size_bytes=len(data), + built_with_model_package={"name": "policyengine-uk", "version": "2.174.1"}, + built_with_core_package={"name": "policyengine-core", "version": "3.33.3"}, + evidence={ + "key": "release_certification", + "filename": "release_certification.json", + "sha256": "1" * 64, + }, + resolved_at="2026-11-10T12:00:00Z", + ) + paths.reference_dir.mkdir(parents=True, exist_ok=True) + paths.reference_h5(reference).write_bytes(data) + write_uk_cut_reference(paths.reference_record, reference) + return reference + + def test_spine_argv_parses(): from microcosm.build.uk_runtime import spine_build @@ -206,11 +235,14 @@ def test_national_argv_parses(tmp_path, dry_run): from microcosm.build.uk_runtime import full_build_cli sources = {s.key: s for s in cut.load_input_sources()} + paths = _paths(tmp_path) + reference = _reference(paths) argv = cut.national_argv( - _paths(tmp_path), + paths, python="py", spine_sha256="b" * 64, sources=sources, + reference=reference, dry_run=dry_run, logbook_prev_row_digest="c" * 64, ) @@ -218,6 +250,9 @@ def test_national_argv_parses(tmp_path, dry_run): assert args.release_role == "national" assert args.dry_run is dry_run assert args.input_sha256 == "b" * 64 + # The build compares its candidate with the cut's own reference. + assert args.reference_h5 == paths.reference_h5(reference) + assert args.reference_record == paths.reference_record @pytest.mark.parametrize("dry_run", [True, False]) @@ -228,11 +263,13 @@ def test_the_national_build_never_publishes_to_the_orrery(tmp_path, dry_run): from microcosm.build.uk_runtime.orrery_publication import publication_enabled sources = {s.key: s for s in cut.load_input_sources()} + paths = _paths(tmp_path) argv = cut.national_argv( - _paths(tmp_path), + paths, python="py", spine_sha256="b" * 64, sources=sources, + reference=_reference(paths), dry_run=dry_run, ) assert "--publish-orrery" not in argv @@ -258,6 +295,8 @@ def test_certify_and_assemble_argv_parse(tmp_path): )[2:] ) assert certify.release_id == cut.RELEASE_ID + assert certify.reference_record == paths.reference_record + assert certify.score_receipt == paths.score_receipt assemble = _tool("assemble_uk_release_dir")._parse_args( cut.assemble_argv(paths, python="py")[2:] ) @@ -265,9 +304,9 @@ def test_certify_and_assemble_argv_parse(tmp_path): def test_score_argv_names_only_flags_the_scorer_declares(tmp_path, capsys): - sources = {s.key: s for s in cut.load_input_sources()} + paths = _paths(tmp_path) argv = cut.score_argv( - _paths(tmp_path), python="py", candidate_sha256="f" * 64, sources=sources + paths, python="py", candidate_sha256="f" * 64, reference=_reference(paths) ) with pytest.raises(SystemExit): _tool("score_uk_national_candidate").main(["--help"]) @@ -513,12 +552,17 @@ def national(self, log_path): (run / "logbook-spool").mkdir() (run / "logbook-spool" / f"{BUILD_ROW}.json").write_text("{}") + def _reference(self) -> dict: + record = json.loads(self.paths.reference_record.read_text()) + return {key: record[key] for key in ("release", "build", "sha256")} + def score(self, log_path): - (self.paths.run_dir / "score_vs_incumbent.json").write_text( + self.paths.score_receipt.write_text( json.dumps( { "artifacts": {"candidate": {"sha256": _sha(b"candidate")}}, - "evaluation": {"verdict": "passed"}, + "reference": self._reference(), + "evaluation": {"schema_version": 2}, } ) ) @@ -529,7 +573,11 @@ def certify(self, log_path): self._report(shippable=True) ) (run / f"{cut.CANDIDATE_NAME}.release_certification.json").write_text( - self._report(shippable=True, candidate={"sha256": _sha(b"candidate")}) + self._report( + shippable=True, + candidate={"sha256": _sha(b"candidate")}, + reference=self._reference(), + ) ) def assemble(self, log_path): @@ -544,8 +592,10 @@ def publish(self, log_path): pass -def _runner(tmp_path, fake=None, signer=None): +def _runner(tmp_path, fake=None, signer=None, *, reference=True): paths = _paths(tmp_path) + if reference: + _reference(paths) fake = fake or FakeBuild(paths, signer=signer) sources = {s.key: s for s in cut.load_input_sources()} runner = cut.Runner( @@ -664,27 +714,62 @@ def test_retrying_the_national_run_keeps_the_failed_attempt(tmp_path): assert len(aside) == 1 and (aside[0] / "evidence.json").read_text() == "kept" -def test_a_losing_candidate_is_not_certified(tmp_path): +def test_a_candidate_that_fits_worse_than_its_reference_is_certified(tmp_path): + """The comparison is a report (microcosm#1095): no loss decides the cut.""" + runner, fake = _runner(tmp_path) + original_score = fake.score + + def worse_score(log_path): + original_score(log_path) + receipt = json.loads(runner.paths.score_receipt.read_text()) + receipt["evaluation"]["losses"] = { + "candidate_full_loss": 0.5, + "reference_full_loss": 0.01, + } + runner.paths.score_receipt.write_text(json.dumps(receipt)) - def failing_score(log_path): - (runner.paths.run_dir / "score_vs_incumbent.json").write_text( + _all(runner) + runner.paths.score_receipt.unlink() + fake.score = worse_score + cut.run_score(runner) + assert cut.score_done(runner.paths)[0] + + +def test_a_comparison_with_another_reference_is_redone_then_refused(tmp_path): + runner, fake = _runner(tmp_path) + cut.run_spine(runner, staging_local_only=True) + cut.run_national(runner, retry=False, logbook_prev_row_digest=None) + + def other_reference_score(log_path): + runner.paths.score_receipt.write_text( json.dumps( { "artifacts": {"candidate": {"sha256": _sha(b"candidate")}}, - "evaluation": {"verdict": "failed"}, + "reference": { + "release": cut.RELEASE_ID, + "build": "x", + "sha256": "0" * 64, + }, + "evaluation": {"schema_version": 2}, } ) ) - fake.score = failing_score - cut.run_spine(runner, staging_local_only=True) - cut.run_national(runner, retry=False, logbook_prev_row_digest=None) - with pytest.raises(cut.CutError, match="does not pass the incumbent score"): + fake.score = other_reference_score + with pytest.raises(cut.CutError, match="another reference"): cut.run_score(runner) assert "certify" not in fake.calls +def test_a_cut_without_a_reference_builds_nothing_that_compares(tmp_path): + runner, fake = _runner(tmp_path, reference=False) + cut.run_spine(runner, staging_local_only=True) + with pytest.raises(cut.CutError, match="no usable reference"): + cut.run_national(runner, retry=False, logbook_prev_row_digest=None) + assert fake.calls == ["spine"] + + @given( prefix=st.lists(st.text(alphabet=st.characters(blacklist_characters="{}"))), tag=st.from_regex(r"[a-z0-9-]{1,40}", fullmatch=True), @@ -762,10 +847,10 @@ def test_retried_score_and_certification_keep_the_earlier_evidence(tmp_path): runner, fake = _runner(tmp_path) cut.run_spine(runner, staging_local_only=True) cut.run_national(runner, retry=False, logbook_prev_row_digest=None) - receipt = runner.paths.run_dir / "score_vs_incumbent.json" - receipt.write_text('{"evaluation": {"verdict": "failed"}}') + receipt = runner.paths.score_receipt + receipt.write_text('{"evaluation": {"schema_version": 1, "verdict": "failed"}}') cut.run_score(runner) - kept = list(runner.paths.run_dir.glob("score_vs_incumbent.json.superseded-*")) + kept = list(runner.paths.run_dir.glob("score_vs_reference.json.superseded-*")) assert len(kept) == 1 and "failed" in kept[0].read_text() certification = ( runner.paths.run_dir / f"{cut.CANDIDATE_NAME}.release_certification.json" @@ -1025,7 +1110,7 @@ def test_the_signing_key_reaches_only_the_steps_that_sign_or_verify( cut.run_spine(runner, staging_local_only=True) cut.run_national(runner, retry=False, logbook_prev_row_digest=None) # Make the score step run, so its environment is seen too. - (runner.paths.run_dir / "score_vs_incumbent.json").unlink(missing_ok=True) + runner.paths.score_receipt.unlink(missing_ok=True) cut.run_score(runner) cut.run_certify(runner) cut.run_assemble(runner) @@ -1165,3 +1250,132 @@ def start(): (check,) = cut.hub_checks() assert not check.ok assert check.detail == "RuntimeError: the emitter's spool is locked" + + +# --------------------------------------------------------------------------- # +# The reference: the current national build, resolved once per cut # +# --------------------------------------------------------------------------- # + + +def _reference_hub(tmp_path): + from test_support.microcosm_build.uk_cut_reference import ( + UK_NATIONAL_RELEASE, + UKReferenceHub, + ) + + hub = UKReferenceHub(tmp_path / "hub-root") + hub.publish(UK_NATIONAL_RELEASE, REFERENCE_BUILD, dataset=b"current build") + return hub + + +def test_the_reference_is_resolved_once_and_kept(tmp_path): + runner, _ = _runner(tmp_path, reference=False) + hub = _reference_hub(tmp_path) + + reference = cut.run_reference(runner, hub_download=hub) + + assert reference.build == REFERENCE_BUILD + assert runner.paths.reference_h5(reference).read_bytes() == b"current build" + assert cut.reference_done(runner.paths)[0] + state = json.loads(runner.paths.state.read_text())["steps"]["reference"] + assert state["status"] == "done" and state["reference"] == reference.label + # A promotion after the fetch moves nothing in this cut. + hub.publish(cut.RELEASE_ID, "uk-frs-calibration-attempt-later", dataset=b"newer") + calls = len(hub.calls) + assert cut.run_reference(runner, hub_download=hub).build == REFERENCE_BUILD + assert len(hub.calls) == calls + + +def test_re_resolving_sets_the_reference_aside_and_moves_to_the_current_build( + tmp_path, +): + runner, _ = _runner(tmp_path, reference=False) + hub = _reference_hub(tmp_path) + cut.run_reference(runner, hub_download=hub) + hub.publish(cut.RELEASE_ID, "uk-frs-calibration-attempt-later", dataset=b"newer") + + reference = cut.run_reference(runner, re_resolve=True, hub_download=hub) + + assert reference.build == "uk-frs-calibration-attempt-later" + kept = list(runner.paths.cut.glob("reference.superseded-*")) + assert len(kept) == 1 and (kept[0] / "reference.json").exists() + + +def test_an_unusable_reference_is_never_resolved_over(tmp_path): + runner, _ = _runner(tmp_path) + runner.paths.reference_h5(cut.cut_reference(runner.paths)).write_bytes(b"x") + with pytest.raises(cut.CutError, match="--re-resolve-reference"): + cut.run_reference(runner, hub_download=_reference_hub(tmp_path)) + + +def test_a_hub_without_a_certified_current_build_stops_the_cut(tmp_path): + from test_support.microcosm_build.uk_cut_reference import UKReferenceHub + + runner, _ = _runner(tmp_path, reference=False) + hub = UKReferenceHub(tmp_path / "hub-root") + hub.publish( + cut.RELEASE_ID, + REFERENCE_BUILD, + dataset=b"current build", + evidence={"kind": "uk_release_certification", "shippable": False}, + ) + with pytest.raises(cut.CutError, match="no reference to compare with"): + cut.run_reference(runner, hub_download=hub) + state = json.loads(runner.paths.state.read_text())["steps"]["reference"] + assert state["status"] == "failed" + + +def test_a_certification_against_another_reference_is_not_done(tmp_path): + runner, _ = _runner(tmp_path) + _all(runner) + assert cut.certify_done(runner.paths)[0] + _reference(runner.paths, build="uk-frs-calibration-attempt-later", data=b"other") + assert cut.score_done(runner.paths) == ( + False, + "score receipt compared with another reference", + ) + assert cut.certify_done(runner.paths) == ( + False, + "certification names another reference", + ) + + +def test_stage1_compares_with_the_cut_reference_by_default(tmp_path): + runner, fake = _runner(tmp_path) + cut.run_spine(runner, staging_local_only=True) + cut.run_national(runner, retry=False, logbook_prev_row_digest=None) + seen = {} + + def stage1(argv, cwd, log_path, env): + seen["argv"] = argv + return 0 + + runner.execute = stage1 + cut.run_stage1(runner, None) + reference = cut.cut_reference(runner.paths) + assert f"previous={runner.paths.reference_h5(reference)}" in seen["argv"] + + +def test_status_says_when_the_current_build_has_moved(tmp_path): + runner, _ = _runner(tmp_path) + out = io.StringIO() + with redirect_stdout(out): + cut.status(runner.paths, check_current=lambda release: {"build": "later"}) + text = out.getvalue() + assert f"done reference: compared with {cut.RELEASE_ID}@{REFERENCE_BUILD}" in text + assert "the current national build is now later" in text + out = io.StringIO() + with redirect_stdout(out): + cut.status( + runner.paths, check_current=lambda release: {"build": REFERENCE_BUILD} + ) + assert "still the current national build" in out.getvalue() + + +def test_the_receipt_records_the_reference(tmp_path): + runner, _ = _runner(tmp_path) + _all(runner) + cut.write_receipt(runner, {"tag": "t", "published": False}) + receipt = json.loads(runner.paths.receipt.read_text()) + assert receipt["reference"]["build"] == REFERENCE_BUILD + assert "incumbent_efrs" not in receipt["inputs"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_score_uk_national_candidate.py index 79fdfca929..74e31f8e80 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_score_uk_national_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_score_uk_national_candidate.py @@ -6,6 +6,10 @@ import pandas as pd import pytest +from microcosm.build.uk_runtime.cut_reference import ( + UKCutReference, + write_uk_cut_reference, +) from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, write_uk_national_frame, @@ -23,6 +27,10 @@ score_uk_national_candidate, ) +REFERENCE_RELEASE = "microcosm-uk-2024-25-national" +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +REFERENCE_LABEL = f"{REFERENCE_RELEASE}@{REFERENCE_BUILD}" + def _registry() -> TargetRegistry: return TargetRegistry( @@ -76,63 +84,101 @@ def _write(path, *, measure_a, measure_b) -> None: write_uk_national_frame(frame, path) +def _reference_record(path, h5) -> UKCutReference: + """The record a cut's fetch writes for ``h5``, written to ``path``.""" + + reference = UKCutReference( + release=REFERENCE_RELEASE, + build=REFERENCE_BUILD, + repo_id="policyengine/populace-uk-private", + path_in_repo=f"releases/{REFERENCE_RELEASE}/{REFERENCE_BUILD}/{h5.name}", + filename=h5.name, + sha256=_sha256_file(h5), + size_bytes=h5.stat().st_size, + built_with_model_package={"name": "policyengine-uk", "version": "2.174.1"}, + built_with_core_package={"name": "policyengine-core", "version": "3.33.3"}, + evidence={ + "key": "release_certification", + "filename": "release_certification.json", + "sha256": "1" * 64, + }, + resolved_at="2026-11-10T12:00:00Z", + ) + write_uk_cut_reference(path, reference) + return reference + + def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) score = score_uk_national_candidate( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE_LABEL, target_registry=_registry(), calibration_year=2025, ) assert score["artifacts"]["candidate"]["label"] == candidate.stem - assert score["artifacts"]["incumbent"]["label"] == "enhanced_frs_2024_25" + assert score["artifacts"]["reference"]["label"] == REFERENCE_LABEL assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) - assert score["artifacts"]["incumbent"]["sha256"] == _sha256_file(incumbent) - assert score["target_drift"] == [ - { - "target": "target_a@0", - "family": "family_a", - "candidate_relative_error": pytest.approx(0.0), - "incumbent_relative_error": pytest.approx(-0.2), - "winner": "candidate", - }, - { - "target": "target_b@0", - "family": "family_b", - "candidate_relative_error": pytest.approx(-0.5), - "incumbent_relative_error": pytest.approx(0.0), - "winner": "incumbent", - }, + assert score["artifacts"]["reference"]["sha256"] == _sha256_file(reference) + compared = score["head_to_head"]["targets"]["compared"] + assert [ + ( + row["name"], + row["family"], + row["candidate_relative_error"], + row["reference_relative_error"], + row["lower_absolute_relative_error"], + ) + for row in compared + ] == [ + ( + "target_a@0", + "family_a", + pytest.approx(0.0), + pytest.approx(-0.2), + "candidate", + ), + ( + "target_b@0", + "family_b", + pytest.approx(-0.5), + pytest.approx(0.0), + "reference", + ), ] assert score["candidate_full_loss"] == pytest.approx(0.25) - assert score["incumbent_full_loss"] == pytest.approx(0.1) + assert score["reference_full_loss"] == pytest.approx(0.1) assert score["candidate_train_loss"] == score["candidate_full_loss"] # With no declared split, the holdout keys report absence rather than # repeating the fitted loss under a name that means generalization. assert score["candidate_holdout_loss"] is None - assert score["incumbent_holdout_loss"] is None + assert score["reference_holdout_loss"] is None assert score["holdout_basis"] == "none_declared" assert score["loss"]["train_equals_full"] is True - assert score["candidate_target_wins"] == 1 - assert score["incumbent_target_wins"] == 1 - assert score["target_wins_by_family"] == { + targets = score["head_to_head"]["targets"] + assert (targets["candidate_lower_count"], targets["reference_lower_count"]) == ( + 1, + 1, + ) + assert score["head_to_head"]["by_family"] == { "family_a": { - "candidate_target_wins": 1, - "incumbent_target_wins": 0, - "ties": 0, + "candidate_lower_count": 1, + "reference_lower_count": 0, + "equal_count": 0, }, "family_b": { - "candidate_target_wins": 0, - "incumbent_target_wins": 1, - "ties": 0, + "candidate_lower_count": 0, + "reference_lower_count": 1, + "equal_count": 0, }, } @@ -140,11 +186,11 @@ def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" registry_json = tmp_path / "registry.json" output_json = tmp_path / "score.json" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) _registry().to_json(registry_json) assert ( @@ -154,10 +200,12 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: str(candidate), "--candidate-sha256", _sha256_file(candidate), - "--incumbent-h5", - str(incumbent), - "--incumbent-sha256", - _sha256_file(incumbent), + "--reference-h5", + str(reference), + "--reference-sha256", + _sha256_file(reference), + "--reference-label", + "explicit_reference", "--registry-json", str(registry_json), "--output-json", @@ -166,8 +214,6 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: "2025", "--candidate-label", "explicit_candidate", - "--incumbent-label", - "explicit_incumbent", "--no-measure-resolution", ] ) @@ -176,30 +222,35 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: score = json.loads(output_json.read_text(encoding="utf-8")) # The receipt is the score block itself: the release-cut certifier reads - # ``artifacts.candidate.sha256`` and ``evaluation.verdict`` at the top - # level, so a wrapper key would leave every CLI receipt unverifiable. - assert "score_vs_enhanced_frs" not in score + # ``artifacts.candidate.sha256`` and ``reference`` at the top level, so a + # wrapper key would leave every CLI receipt unverifiable. + assert "score_vs_reference" not in score assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) - # Raw-column scoring has nothing to prune; the surface still closes and - # the verdict is decided: this candidate fits worse than the incumbent. - assert score["incumbent_unresolvable_pruned"]["n_pruned"] == 0 + # Raw-column scoring has nothing to prune; the surface still closes, and + # a candidate that fits worse than the reference is reported, not refused. + assert score["reference_unresolvable_pruned"]["n_pruned"] == 0 assert score["evaluation"]["scored_surface"] == { "n_scored": 2, "n_pruned": 0, "n_surface": 2, } - assert score["evaluation"]["rule_1"]["passed"] is False - assert score["evaluation"]["verdict"] == "failed" + assert "verdict" not in score["evaluation"] + assert ( + score["evaluation"]["losses"]["candidate_full_loss"] + > (score["evaluation"]["losses"]["reference_full_loss"]) + ) + # An ad hoc comparison carries no record, so it never certifies a cut. + assert score["reference"] is None assert score["artifacts"]["candidate"]["label"] == "explicit_candidate" - assert score["artifacts"]["incumbent"]["label"] == "explicit_incumbent" + assert score["artifacts"]["reference"]["label"] == "explicit_reference" assert score["holdout_basis"] == "none_declared" # An undeclared holdout reports absence, never the fitted loss wearing a # holdout name: June's fixture holds a genuinely different holdout value, # so a copied one would read as generalization that was never measured. assert score["candidate_holdout_loss"] is None - assert score["incumbent_holdout_loss"] is None + assert score["reference_holdout_loss"] is None assert isinstance(score["candidate_train_loss"], float) - assert isinstance(score["incumbent_full_loss"], float) + assert isinstance(score["reference_full_loss"], float) assert _load_registry(registry_json).version == _registry().version @@ -208,14 +259,16 @@ def test_verify_score_receipt_accepts_the_cli_receipt(tmp_path) -> None: pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" registry_json = tmp_path / "registry.json" - output_json = tmp_path / "score_vs_incumbent.json" - # The candidate hits both targets; the incumbent misses both. + output_json = tmp_path / "score_vs_reference.json" + record_json = tmp_path / "reference.json" + # The candidate hits both targets; the reference misses both. _write(candidate, measure_a=[5.0, 5.0], measure_b=[10.0, 10.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[6.0, 6.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[6.0, 6.0]) _registry().to_json(registry_json) candidate_sha = _sha256_file(candidate) + reference_record = _reference_record(record_json, reference) assert ( main( @@ -224,10 +277,10 @@ def test_verify_score_receipt_accepts_the_cli_receipt(tmp_path) -> None: str(candidate), "--candidate-sha256", candidate_sha, - "--incumbent-h5", - str(incumbent), - "--incumbent-sha256", - _sha256_file(incumbent), + "--reference-h5", + str(reference), + "--reference-record", + str(record_json), "--registry-json", str(registry_json), "--output-json", @@ -241,25 +294,63 @@ def test_verify_score_receipt_accepts_the_cli_receipt(tmp_path) -> None: ) receipt = json.loads(output_json.read_text(encoding="utf-8")) - assert receipt["evaluation"]["verdict"] == "passed" - _verify_score_receipt(receipt, candidate_sha256=candidate_sha) + assert receipt["evaluation"]["reference"] == REFERENCE_LABEL + assert receipt["reference"] == reference_record.to_record() + _verify_score_receipt( + receipt, candidate_sha256=candidate_sha, reference=reference_record + ) with pytest.raises(UKReleaseCertificationError, match="artifacts.candidate.sha256"): - _verify_score_receipt(receipt, candidate_sha256="0" * 64) + _verify_score_receipt( + receipt, candidate_sha256="0" * 64, reference=reference_record + ) + other = UKCutReference.from_record( + {**reference_record.to_record(), "build": "uk-frs-calibration-attempt-other"} + ) + with pytest.raises(UKReleaseCertificationError, match="another reference"): + _verify_score_receipt(receipt, candidate_sha256=candidate_sha, reference=other) + + +def test_the_cli_takes_a_record_or_an_ad_hoc_label_not_both(tmp_path, capsys) -> None: + candidate = tmp_path / "candidate.h5" + reference = tmp_path / "reference.h5" + candidate.write_bytes(b"c") + reference.write_bytes(b"r") + base = [ + "--candidate-h5", str(candidate), + "--candidate-sha256", "0" * 64, + "--reference-h5", str(reference), + "--registry-json", str(tmp_path / "registry.json"), + "--output-json", str(tmp_path / "out.json"), + ] # fmt: skip + with pytest.raises(SystemExit): + main(base) + assert "--reference-record" in capsys.readouterr().err + with pytest.raises(SystemExit): + main( + [ + *base, + "--reference-record", + str(tmp_path / "reference.json"), + "--reference-label", + "x", + ] # fmt: skip + ) def test_scorer_refuses_artifacts_that_do_not_match_their_pins(tmp_path) -> None: pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) with pytest.raises(ValueError, match="candidate artifact sha mismatch"): score_uk_national_candidate( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256="0" * 64, - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE_LABEL, target_registry=_registry(), calibration_year=2025, ) @@ -267,7 +358,7 @@ def test_scorer_refuses_artifacts_that_do_not_match_their_pins(tmp_path) -> None def test_scorer_refuses_a_register_that_is_not_a_validated_artifact(tmp_path) -> None: # The loose loader accepted any object with a specs list, so a - # hand-edited surface could decide rule 1 unnoticed. + # hand-edited surface could shape the comparison unnoticed. loose = tmp_path / "loose.json" loose.write_text( json.dumps( @@ -300,9 +391,9 @@ def test_scorer_refuses_prepared_measures_it_cannot_materialize(tmp_path) -> Non pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) production_shaped = TargetRegistry( [ TargetSpec( @@ -321,9 +412,10 @@ def test_scorer_refuses_prepared_measures_it_cannot_materialize(tmp_path) -> Non with pytest.raises(RuntimeError, match="did not materialize for scoring"): score_uk_national_candidate( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE_LABEL, target_registry=production_shaped, calibration_year=2025, ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_candidate_score.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_candidate_score.py index a6fd6f0915..17059303ae 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_candidate_score.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_candidate_score.py @@ -1,7 +1,9 @@ -"""The UK candidate score on the common resolvable surface (microcosm#967). +"""The UK head-to-head with the reference on the common resolvable surface. -The pruning loop is exercised against a stand-in for the scoring frame that -fails the incumbent's resolution the way the real resolver does — a +microcosm#967 built the common surface; microcosm#1095 made the reference the +current national build and the comparison a report with no verdict. The +pruning loop is exercised against a stand-in for the scoring frame that +fails the reference's resolution the way the real resolver does — a ``MeasureResolutionError`` naming the absent measure, with the resolution receipt's skip rows — while both frames score on raw columns, so the suite needs no engine. @@ -21,9 +23,10 @@ from microcosm.build.uk_runtime import candidate_score from microcosm.build.uk_runtime.candidate_score import ( _sha256_file, - evaluate_uk_candidate_against_incumbent, + evaluate_uk_candidate_against_reference, evaluation_block, - prune_incumbent_unresolvable, + load_uk_reference_unresolvable_measures, + prune_reference_unresolvable, pruned_warning, ) from microcosm.build.uk_runtime.national_frame import ( @@ -39,6 +42,7 @@ from microcosm.frame import WeightKind MEASURE_B = "household.measure_b" +REFERENCE = "microcosm-uk-2024-25-national@uk-frs-calibration-attempt-test" def _registry() -> TargetRegistry: @@ -102,16 +106,16 @@ def _write( def _twins(tmp_path: Path) -> tuple[Path, Path]: candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" # The candidate hits target_a exactly and carries measure_b; the - # incumbent misses target_a and never carried measure_b. + # reference misses target_a and never carried measure_b. _write(candidate, measure_a=[5.0, 5.0], measure_b=[10.0, 10.0]) - _write(incumbent, measure_a=[4.0, 4.0]) - return candidate, incumbent + _write(reference, measure_a=[4.0, 4.0]) + return candidate, reference def _resolver_factory(path, frame): - # A resolver is present (so the incumbent is probed) but scoring stays + # A resolver is present (so the reference is probed) but scoring stays # on raw columns: the real resolver is stood in for by ``_intercept``. return None @@ -130,14 +134,14 @@ def fake( factory, *, band_edge_registry=None, - incumbent=False, + reference=False, ): calls.append( { "h5": Path(h5_path), "targets": [spec.name for spec in registry.specs], "band_edge_registry": band_edge_registry, - "incumbent": incumbent, + "reference": reference, } ) if Path(h5_path) == failing_h5 and any( @@ -152,7 +156,7 @@ def fake( _REVIEWED = { "household.measure_b": UKReviewedExclusion( - reason="the incumbent twin never carried measure_b", + reason="the reference twin never carried measure_b", approved_by="juaristi22", adjudication="microcosm#823 (test register)", approved_on="2026-09-01", @@ -166,10 +170,11 @@ def _reviewed_register(monkeypatch): """Stand in for the packaged register: measure_b is a signed, in-force entry. Tests that need another register override the loader again.""" + def reviewed(*, reference_label, source=None): + return dict(_REVIEWED) if reference_label == REFERENCE else {} + monkeypatch.setattr( - candidate_score, - "load_uk_incumbent_unresolvable_measures", - lambda source=None: dict(_REVIEWED), + candidate_score, "load_uk_reference_unresolvable_measures", reviewed ) @@ -180,30 +185,31 @@ def _reviewed_register(monkeypatch): } -def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( +def test_prunes_the_target_the_reference_cannot_materialize_and_scores_the_rest( monkeypatch, tmp_path ): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) full = _registry() calls = _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message=f"provider does not know {MEASURE_B}", receipt=_SKIP_RECEIPT, ) - score = evaluate_uk_candidate_against_incumbent( + score = evaluate_uk_candidate_against_reference( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE, target_registry=full, calibration_year=2025, measure_resolver_factory=_resolver_factory, ) - pruned = score["incumbent_unresolvable_pruned"] + pruned = score["reference_unresolvable_pruned"] assert pruned["n_pruned"] == 1 assert pruned["n_scored"] == 1 assert pruned["n_surface"] == 2 @@ -218,10 +224,13 @@ def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( } # Both arms were scored on the common surface only. assert score["register"]["n_specs"] == 1 - assert [row["target"] for row in score["target_drift"]] == ["target_a@0"] - # The candidate hit its target and the incumbent missed: rule 1 passes. - assert score["evaluation"]["verdict"] == "passed" - assert score["evaluation"]["rule_1"]["passed"] is True + compared = score["head_to_head"]["targets"]["compared"] + assert [row["name"] for row in compared] == ["target_a@0"] + # The candidate hit its target and the reference missed; the comparison + # reports it and decides nothing. + assert compared[0]["lower_absolute_relative_error"] == "candidate" + assert "verdict" not in score["evaluation"] + assert score["evaluation"]["reference"] == REFERENCE assert score["evaluation"]["scored_surface"] == { "n_scored": 1, "n_pruned": 1, @@ -233,9 +242,9 @@ def test_prunes_the_target_the_incumbent_cannot_materialize_and_scores_the_rest( assert scoring_calls and all( call["band_edge_registry"] is full for call in scoring_calls ) - # Only the incumbent may read its current income for the annual measures. + # Only the reference may read its current income for the annual measures. assert calls and all( - call["incumbent"] == (call["h5"] == incumbent) for call in calls + call["reference"] == (call["h5"] == reference) for call in calls ) warning = pruned_warning(score) assert warning is not None @@ -247,10 +256,10 @@ def test_binding_fallback_names_the_target_when_the_receipt_does_not( monkeypatch, tmp_path ): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message=f"provider failed computing {MEASURE_B}: no such column", receipt={"skips": []}, ) @@ -258,8 +267,13 @@ def test_binding_fallback_names_the_target_when_the_receipt_does_not( "test.target_b": {"bindings": {"policyengine": {"value_variable": "measure_b"}}} } - surface, pruned = prune_incumbent_unresolvable( - incumbent, _registry(), 2025, _resolver_factory, contract_targets=contract + surface, pruned = prune_reference_unresolvable( + reference, + _registry(), + 2025, + _resolver_factory, + contract_targets=contract, + reference_label=REFERENCE, ) assert [spec.name for spec in surface.specs] == ["target_a"] @@ -268,65 +282,76 @@ def test_binding_fallback_names_the_target_when_the_receipt_does_not( def test_refuses_to_prune_blindly(monkeypatch, tmp_path): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message=f"provider does not know {MEASURE_B}", receipt={"skips": []}, ) with pytest.raises(MeasureResolutionError, match="refusing to prune blindly"): - prune_incumbent_unresolvable( - incumbent, _registry(), 2025, _resolver_factory, contract_targets={} + prune_reference_unresolvable( + reference, + _registry(), + 2025, + _resolver_factory, + contract_targets={}, + reference_label=REFERENCE, ) def test_a_failure_naming_no_measure_is_re_raised(monkeypatch, tmp_path): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message="measure materialization made no progress on round 0", receipt={"skips": []}, ) with pytest.raises(MeasureResolutionError, match="made no progress"): - prune_incumbent_unresolvable( - incumbent, _registry(), 2025, _resolver_factory, contract_targets={} + prune_reference_unresolvable( + reference, + _registry(), + 2025, + _resolver_factory, + contract_targets={}, + reference_label=REFERENCE, ) def test_pruning_off_restores_the_refusal(monkeypatch, tmp_path): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message=f"provider does not know {MEASURE_B}", receipt=_SKIP_RECEIPT, ) with pytest.raises(MeasureResolutionError, match="does not know"): - evaluate_uk_candidate_against_incumbent( + evaluate_uk_candidate_against_reference( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE, target_registry=_registry(), calibration_year=2025, measure_resolver_factory=_resolver_factory, - prune_incumbent_unresolvable_measures=False, + prune_reference_unresolvable_measures=False, ) def test_a_candidate_side_gap_still_refuses(monkeypatch, tmp_path): pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) _intercept( monkeypatch, failing_h5=candidate, @@ -334,47 +359,84 @@ def test_a_candidate_side_gap_still_refuses(monkeypatch, tmp_path): receipt=_SKIP_RECEIPT, ) - # Only the incumbent is probed for pruning; a measure the candidate + # Only the reference is probed for pruning; a measure the candidate # cannot materialize is a defect and refuses as before. with pytest.raises(MeasureResolutionError, match="does not know"): - evaluate_uk_candidate_against_incumbent( + evaluate_uk_candidate_against_reference( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE, target_registry=_registry(), calibration_year=2025, measure_resolver_factory=_resolver_factory, ) -def test_verdict_fails_when_the_incumbent_fits_better(tmp_path): +def test_a_reference_that_fits_better_is_reported_not_refused(tmp_path): pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) - score = evaluate_uk_candidate_against_incumbent( + score = evaluate_uk_candidate_against_reference( candidate_h5=candidate, - incumbent_h5=incumbent, + reference_h5=reference, candidate_sha256=_sha256_file(candidate), - incumbent_sha256=_sha256_file(incumbent), + reference_sha256=_sha256_file(reference), + reference_label=REFERENCE, target_registry=_registry(), calibration_year=2025, ) - assert score["incumbent_unresolvable_pruned"]["n_pruned"] == 0 + assert score["reference_unresolvable_pruned"]["n_pruned"] == 0 assert pruned_warning(score) is None - assert score["evaluation"]["scored_surface"] == { + evaluation = score["evaluation"] + assert evaluation["schema_version"] == 2 + assert evaluation["scored_surface"] == { "n_scored": 2, "n_pruned": 0, "n_surface": 2, } - assert score["evaluation"]["rule_1"]["candidate_full_loss"] == pytest.approx(0.25) - assert score["evaluation"]["rule_1"]["incumbent_full_loss"] == pytest.approx(0.1) - assert score["evaluation"]["verdict"] == "failed" - assert evaluation_block(score) == score["evaluation"] + assert evaluation["losses"]["candidate_full_loss"] == pytest.approx(0.25) + assert evaluation["losses"]["reference_full_loss"] == pytest.approx(0.1) + assert evaluation["targets"] == { + "candidate_lower_count": 1, + "reference_lower_count": 1, + "equal_count": 0, + } + assert evaluation["no_threshold_applied"] is True + assert "verdict" not in evaluation + assert score["head_to_head"]["by_family"] == { + "family_a": { + "candidate_lower_count": 1, + "reference_lower_count": 0, + "equal_count": 0, + }, + "family_b": { + "candidate_lower_count": 0, + "reference_lower_count": 1, + "equal_count": 0, + }, + } + assert evaluation_block(score) == evaluation + assert score["reference"] is None + + +def test_the_reference_record_must_name_the_scored_file(tmp_path): + pytest.importorskip("tables") + candidate, reference = _twins(tmp_path) + _write(reference, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + release, build = REFERENCE.split("@") + record = {"release": release, "build": build, "sha256": _sha256_file(reference)} + + score = _evaluate(candidate, reference, reference_record=record) + assert score["reference"] == record + + with pytest.raises(ValueError, match="names another build"): + _evaluate(candidate, reference, reference_record={**record, "sha256": "0" * 64}) def test_each_file_is_scored_with_the_weights_the_engine_gives_it(tmp_path): @@ -395,40 +457,42 @@ def test_each_file_is_scored_with_the_weights_the_engine_gives_it(tmp_path): _write(as_solved, measure_a=[5.0, 5.0], measure_b=[10.0, 10.0], time_period="2024") factor = uk_weight_projection_factor(2024, 2025) - score = evaluate_uk_candidate_against_incumbent( + score = evaluate_uk_candidate_against_reference( candidate_h5=released, - incumbent_h5=as_solved, + reference_h5=as_solved, candidate_sha256=_sha256_file(released), - incumbent_sha256=_sha256_file(as_solved), + reference_sha256=_sha256_file(as_solved), + reference_label=REFERENCE, target_registry=_registry(), calibration_year=2025, ) - rule_1 = score["evaluation"]["rule_1"] + losses = score["evaluation"]["losses"] # The release reads back its solved weights; the as-solved file reads # high on every row by the population growth of the year after its own. - assert rule_1["candidate_full_loss"] == pytest.approx(0.0, abs=1e-12) - assert rule_1["incumbent_full_loss"] == pytest.approx(factor - 1.0, rel=1e-9) + assert losses["candidate_full_loss"] == pytest.approx(0.0, abs=1e-12) + assert losses["reference_full_loss"] == pytest.approx(factor - 1.0, rel=1e-9) side = {"dataset_year": 2024, "engine_projection_factor": factor} assert score["weight_year"] == { "scored_year": 2025, "candidate": side, - "incumbent": side, + "reference": side, } -def _evaluate(candidate, incumbent, **overrides): +def _evaluate(candidate, reference, **overrides): kwargs = { "candidate_h5": candidate, - "incumbent_h5": incumbent, + "reference_h5": reference, "candidate_sha256": _sha256_file(candidate), - "incumbent_sha256": _sha256_file(incumbent), + "reference_sha256": _sha256_file(reference), + "reference_label": REFERENCE, "target_registry": _registry(), "calibration_year": 2025, "measure_resolver_factory": _resolver_factory, } kwargs.update(overrides) - return candidate_score.evaluate_uk_candidate_against_incumbent(**kwargs) + return candidate_score.evaluate_uk_candidate_against_reference(**kwargs) def test_an_unlisted_unresolvable_measure_refuses_instead_of_pruning( @@ -436,52 +500,55 @@ def test_an_unlisted_unresolvable_measure_refuses_instead_of_pruning( ): """Pruning is doctrine, not inference: no signed entry, no prune.""" pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message="provider does not know household.measure_b", receipt=_SKIP_RECEIPT, ) monkeypatch.setattr( - candidate_score, "load_uk_incumbent_unresolvable_measures", lambda s=None: {} + candidate_score, + "load_uk_reference_unresolvable_measures", + lambda *, reference_label, source=None: {}, ) - with pytest.raises(MeasureResolutionError, match="not on the reviewed"): - _evaluate(candidate, incumbent) + with pytest.raises(MeasureResolutionError, match="reference-unresolvable register"): + _evaluate(candidate, reference) def test_an_expired_or_premature_entry_refuses(monkeypatch, tmp_path): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message="provider does not know household.measure_b", receipt=_SKIP_RECEIPT, ) with pytest.raises(MeasureResolutionError, match="expired"): - _evaluate(candidate, incumbent, evaluated_on=date(2100, 1, 1)) + _evaluate(candidate, reference, evaluated_on=date(2100, 1, 1)) with pytest.raises(MeasureResolutionError, match="not yet in force"): - _evaluate(candidate, incumbent, evaluated_on=date(2026, 8, 1)) + _evaluate(candidate, reference, evaluated_on=date(2026, 8, 1)) def test_pruned_rows_carry_the_register_entry_they_stand_on(monkeypatch, tmp_path): pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message="provider does not know household.measure_b", receipt=_SKIP_RECEIPT, ) - score = _evaluate(candidate, incumbent, evaluated_on=date(2026, 9, 22)) - pruned = score["incumbent_unresolvable_pruned"] + score = _evaluate(candidate, reference, evaluated_on=date(2026, 9, 22)) + pruned = score["reference_unresolvable_pruned"] assert pruned["measures"] == ["household.measure_b"] assert pruned["pruned_targets"]["target_b"]["adjudication"] == ( "microcosm#823 (test register)" ) register = pruned["reviewed_register"] - assert register["resource"] == "incumbent_unresolvable_measures.json" + assert register["resource"] == "reference_unresolvable_measures.json" + assert register["reference"] == REFERENCE assert len(register["sha256"]) == 64 assert register["entries_used"] == { "household.measure_b": _REVIEWED["household.measure_b"].policy_payload() @@ -493,10 +560,10 @@ def test_a_skip_reason_is_matched_exactly_never_by_substring(monkeypatch, tmp_pa with no skip naming the measure and no binding, the loop refuses to prune blindly rather than dropping an unrelated row.""" pytest.importorskip("tables") - candidate, incumbent = _twins(tmp_path) + candidate, reference = _twins(tmp_path) _intercept( monkeypatch, - failing_h5=incumbent, + failing_h5=reference, message="provider does not know household.measure_b", receipt={ "skips": [ @@ -509,20 +576,20 @@ def test_a_skip_reason_is_matched_exactly_never_by_substring(monkeypatch, tmp_pa }, ) with pytest.raises(MeasureResolutionError, match="refusing to prune blindly"): - _evaluate(candidate, incumbent, contract_targets={}) + _evaluate(candidate, reference, contract_targets={}) def test_a_gap_shared_by_both_artifacts_still_refuses_on_the_candidate( monkeypatch, tmp_path ): """Review A1: the candidate is validated on the full surface BEFORE the - incumbent's gaps are pruned, so an export missing a listed measure is a - defect even when the incumbent lacks it too; it never scores as a win.""" + reference's gaps are pruned, so an export missing a listed measure is a + defect even when the reference lacks it too; it never scores as a win.""" pytest.importorskip("tables") candidate = tmp_path / "candidate.h5" - incumbent = tmp_path / "incumbent.h5" + reference = tmp_path / "reference.h5" _write(candidate, measure_a=[5.0, 5.0]) - _write(incumbent, measure_a=[4.0, 4.0]) + _write(reference, measure_a=[4.0, 4.0]) original = candidate_score._scored_frame calls: list[tuple[Path, list[str]]] = [] @@ -533,7 +600,7 @@ def fake( factory, *, band_edge_registry=None, - incumbent=False, + reference=False, ): calls.append((Path(h5_path), [spec.name for spec in registry.specs])) if any(spec.name == "target_b" for spec in registry.specs): @@ -545,8 +612,46 @@ def fake( monkeypatch.setattr(candidate_score, "_scored_frame", fake) with pytest.raises(MeasureResolutionError, match="does not know"): - _evaluate(candidate, incumbent, evaluated_on=date(2026, 9, 22)) - # The first probe was the candidate on the full surface; the incumbent + _evaluate(candidate, reference, evaluated_on=date(2026, 9, 22)) + # The first probe was the candidate on the full surface; the reference # was never asked, so nothing was pruned. assert calls[0] == (candidate, ["target_a", "target_b"]) assert all(path == candidate for path, _ in calls) + + +def test_the_register_lends_each_reference_only_its_own_entries(tmp_path): + """An entry signed for one build is dormant for every other reference.""" + + entry = { + "reason": "the earlier build never carried measure_b", + "approved_by": "juaristi22", + "adjudication": "microcosm#1095 (test register)", + "approved_on": "2026-11-10", + "expires_on": "2099-12-31", + } + register = tmp_path / "reference_unresolvable_measures.json" + register.write_text( + json.dumps( + { + "schema_version": 3, + "description": "test register", + "references": {REFERENCE: {MEASURE_B: entry}}, + } + ) + ) + + mine = load_uk_reference_unresolvable_measures( + reference_label=REFERENCE, source=register + ) + other = load_uk_reference_unresolvable_measures( + reference_label="microcosm-uk-2024-25-national@another-build", source=register + ) + + assert list(mine) == [MEASURE_B] + assert other == {} + + +def test_the_packaged_register_starts_empty(): + """The enhanced FRS's eight entries retired with it (microcosm#1095).""" + + assert candidate_score.load_uk_reference_unresolvable_register() == {} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py index 1e898dd769..671495bb60 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py @@ -35,7 +35,8 @@ def test_certifier_pins_carry_the_byte_sizes_the_logbook_digest_requires( "build_record.json", "terminal_gates.json", "input_mass_reference.json", - "score_vs_incumbent.json", + "score_vs_reference.json", + "reference.json", ): placeholders[name] = tmp_path / name placeholders[name].write_text("{}") @@ -78,7 +79,9 @@ def fake_run(args, state): "--input-mass-reference", str(placeholders["input_mass_reference.json"]), "--score-receipt", - str(placeholders["score_vs_incumbent.json"]), + str(placeholders["score_vs_reference.json"]), + "--reference-record", + str(placeholders["reference.json"]), "--release-id", "microcosm-uk-2024-25-national", "--release-cut-gate-json", @@ -146,7 +149,12 @@ def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( ledger = tmp_path / "ledger" ledger.mkdir() (ledger / "consumer_facts.jsonl").write_text('{"fact": 1}\n') - for name in ("terminal_gates.json", "input_mass_reference.json", "score.json"): + for name in ( + "terminal_gates.json", + "input_mass_reference.json", + "score.json", + "reference.json", + ): (tmp_path / name).write_text("{}") monkeypatch.setattr(certify, "resolve_predecessor", lambda digest: None) monkeypatch.setattr(certify, "record_terminal_attempt", lambda **kwargs: None) @@ -180,6 +188,8 @@ def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( str(tmp_path / "input_mass_reference.json"), "--score-receipt", str(tmp_path / "score.json"), + "--reference-record", + str(tmp_path / "reference.json"), "--release-id", "microcosm-uk-2024-25-national", "--release-cut-gate-json", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 9e78002cee..a4e63e6449 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -104,7 +104,7 @@ def test_release_role_is_required_and_supplies_the_dense_posture(tmp_path): [ (["--target-loss-cap", "5"], "--target-loss-cap"), (["--allow-unpinned-feed"], "--allow-unpinned-feed"), - (["--incumbent-h5", "incumbent.h5"], "--incumbent-h5"), + (["--reference-h5", "reference.h5"], "--reference-h5"), (["--target-weight-rule", "family_equal"], "--target-weight-rule family_equal"), ], ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_incumbent_extractor.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_incumbent_extractor.py index 7255ae3532..acfc8cfeba 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_incumbent_extractor.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_incumbent_extractor.py @@ -171,7 +171,7 @@ def missing_uc_inputs(resolver, grain, *, period, household_ids): monkeypatch.setattr(tool, "compute_household_metrics", missing_uc_inputs) with pytest.raises( - ValueError, match="incumbent_unresolvable_measures.json" + ValueError, match="does not prune unresolvable measures" ) as raised: tool.extract_incumbent_surface( incumbent_h5=incumbent, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_assembler.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_assembler.py index ee95583ccc..f12a42c039 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_assembler.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_assembler.py @@ -435,7 +435,7 @@ def test_assemble_green_release_dir(assembler_inputs, capsys) -> None: "terminal_gates": "terminal_gates", "release_cut_gates": "release_cut_gates", "spine_gates": "spine_gates", - "score_vs_enhanced_frs": "score_vs_enhanced_frs", + "score_vs_reference": "score_vs_reference", } assert "dataset_role" not in release_manifest # The spine ran the SPI stages, so the release says it rests on the tape, @@ -666,7 +666,7 @@ def test_assemble_refuses_invalid_staging_delivery( def test_assemble_refuses_missing_score_receipt(assembler_inputs) -> None: assembler_inputs["score"].unlink() - with pytest.raises(FileNotFoundError, match="score_vs_enhanced_frs"): + with pytest.raises(FileNotFoundError, match="score_vs_reference"): _load_driver_module().main(assembler_inputs["argv"]) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py index 5baac52743..9a48a15dc0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py @@ -50,6 +50,8 @@ def _load_fixture_module(): _TEST_KEY = _FIXTURES.TEST_KEY _green_certification_inputs_fixture = _FIXTURES.green_certification_inputs green_score_receipt = _FIXTURES.green_score_receipt +GREEN_REFERENCE_RECORD = _FIXTURES.GREEN_REFERENCE_RECORD +GREEN_REFERENCE_LABEL = _FIXTURES.GREEN_REFERENCE_LABEL _signing_key_fixture = _FIXTURES.signing_key _sha = _FIXTURES.sha256 _stub_registry = _FIXTURES.stub_registry @@ -380,7 +382,7 @@ def test_compose_refuses_score_receipt_scored_on_another_artifact( { "artifacts": { "candidate": {"sha256": "8" * 64}, - "incumbent": {"sha256": candidate_sha}, + "reference": {"sha256": candidate_sha}, }, "provenance": {"inputs": [candidate_sha]}, } @@ -391,30 +393,66 @@ def test_compose_refuses_score_receipt_scored_on_another_artifact( compose_uk_release_certification(**green_certification_inputs) -def test_compose_refuses_a_score_receipt_whose_verdict_is_not_passed( +def test_compose_refuses_a_receipt_measured_against_another_reference( green_certification_inputs, ): + """The comparison must be with the cut's own reference (microcosm#1095).""" receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) - receipt["evaluation"]["verdict"] = "failed" - receipt["evaluation"]["rule_1"]["passed"] = False + receipt["reference"]["build"] = ( + "uk-frs-calibration-attempt-20261025T091414Z-a6299504" + ) + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="another reference"): + compose_uk_release_certification(**green_certification_inputs) + + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["artifacts"]["reference"]["sha256"] = "7" * 64 green_certification_inputs["score_receipt_path"].write_text( json.dumps(receipt), encoding="utf-8" ) - with pytest.raises(UKReleaseCertificationError, match="verdict is 'failed'"): + with pytest.raises(UKReleaseCertificationError, match="another reference"): compose_uk_release_certification(**green_certification_inputs) +def test_compose_certifies_a_candidate_that_fits_worse_than_its_reference( + green_certification_inputs, +): + """The comparison has no threshold: it is evidence, never a refusal.""" + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["evaluation"]["losses"] = { + "candidate_full_loss": 0.5, + "reference_full_loss": 0.01, + } + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + certification = compose_uk_release_certification(**green_certification_inputs) + assert certification["shippable"] is True + assert certification["score_receipt"]["evaluation"]["candidate_full_loss"] == 0.5 + + def test_compose_refuses_a_score_receipt_without_an_evaluation( green_certification_inputs, ): - # A receipt from the strict scorer carries the cross-pin but no verdict: - # rule 1 was never decided on a closed surface, so it cannot certify. + # A receipt without the head-to-head summary, or one scored against the + # enhanced FRS under the old strict rule, was not made against the cut's + # reference, so it cannot certify. receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) del receipt["evaluation"] green_certification_inputs["score_receipt_path"].write_text( json.dumps(receipt), encoding="utf-8" ) - with pytest.raises(UKReleaseCertificationError, match="no evaluation block"): + with pytest.raises(UKReleaseCertificationError, match="no schema-2 evaluation"): + compose_uk_release_certification(**green_certification_inputs) + + receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) + receipt["evaluation"] = {"schema_version": 1, "verdict": "passed"} + green_certification_inputs["score_receipt_path"].write_text( + json.dumps(receipt), encoding="utf-8" + ) + with pytest.raises(UKReleaseCertificationError, match="no schema-2 evaluation"): compose_uk_release_certification(**green_certification_inputs) @@ -434,9 +472,9 @@ def test_compose_refuses_a_score_receipt_pruned_through_an_unlisted_measure( green_certification_inputs, ): """Pruning from both arms is doctrine: a measure the scorer pruned must - carry a signed entry on the reviewed incumbent-unresolvable register.""" + carry a signed entry on the reviewed reference-unresolvable register.""" receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) - receipt["incumbent_unresolvable_pruned"].update( + receipt["reference_unresolvable_pruned"].update( {"n_pruned": 1, "n_scored": 306, "measures": ["benunit.not_reviewed"]} ) receipt["evaluation"]["scored_surface"].update({"n_pruned": 1, "n_scored": 306}) @@ -451,7 +489,7 @@ def test_compose_refuses_a_score_receipt_pruned_through_an_unlisted_measure( def _listed(approved_on: str, expires_on: str): return { "benunit.not_reviewed": UKReviewedExclusion( - reason="the incumbent never carried it", + reason="the reference never carried it", approved_by="juaristi22", adjudication="microcosm#823 (test)", approved_on=approved_on, @@ -496,17 +534,29 @@ def test_compose_refuses_a_build_record_without_a_recorded_parent( compose_uk_release_certification(**green_certification_inputs) -def test_certification_summarises_the_score_receipt_verdict( +def test_certification_names_its_reference_and_summarises_the_comparison( green_certification_inputs, ): certification = compose_uk_release_certification(**green_certification_inputs) + assert certification["schema_version"] == 2 + assert certification["reference"] == { + "release": GREEN_REFERENCE_RECORD["release"], + "build": GREEN_REFERENCE_RECORD["build"], + "filename": GREEN_REFERENCE_RECORD["filename"], + "sha256": GREEN_REFERENCE_RECORD["sha256"], + "built_with_model_package": GREEN_REFERENCE_RECORD["built_with_model_package"], + } + assert certification["score_receipt"]["filename"] == "score_vs_reference.json" assert certification["score_receipt"]["evaluation"] == { - "verdict": "passed", + "reference": GREEN_REFERENCE_LABEL, "n_scored": 307, "n_pruned": 0, "n_surface": 307, "candidate_full_loss": 0.0096, - "incumbent_full_loss": 0.211, + "reference_full_loss": 0.0101, + "candidate_lower_count": 293, + "reference_lower_count": 12, + "equal_count": 2, } @@ -596,33 +646,44 @@ def test_certification_fields_mirror_the_data_contract(green_certification_input def test_compose_refuses_a_receipt_scored_under_another_register( - green_certification_inputs, + green_certification_inputs, monkeypatch ): """Against the committed register of record, the receipt's register digest must be the committed one (review A2, second half).""" - from datetime import date + from microcosm.build.uk_runtime import candidate_score + from microcosm.build.uk_runtime.weighted_integrity import UKReviewedExclusion - from microcosm.build.uk_runtime.candidate_score import ( - load_uk_incumbent_unresolvable_measures, - uk_incumbent_unresolvable_register_digest, + measure = "benunit.reviewed_for_this_reference" + monkeypatch.setattr( + candidate_score, + "load_uk_reference_unresolvable_measures", + lambda *, reference_label, source=None: ( + { + measure: UKReviewedExclusion( + reason="the reference never carried it", + approved_by="juaristi22", + adjudication="microcosm#1095 (test)", + approved_on="2026-08-01", + expires_on="2027-03-01", + ) + } + if reference_label == GREEN_REFERENCE_LABEL + else {} + ), ) - - measure = sorted(load_uk_incumbent_unresolvable_measures())[0] receipt = green_score_receipt(green_certification_inputs["candidate_sha256"]) - receipt["incumbent_unresolvable_pruned"].update( + receipt["reference_unresolvable_pruned"].update( {"n_pruned": 1, "n_scored": 306, "measures": [measure]} ) - receipt["incumbent_unresolvable_pruned"]["reviewed_register"]["sha256"] = "0" * 64 + receipt["reference_unresolvable_pruned"]["reviewed_register"]["sha256"] = "0" * 64 receipt["evaluation"]["scored_surface"].update({"n_pruned": 1, "n_scored": 306}) path = green_certification_inputs["score_receipt_path"] path.write_text(json.dumps(receipt), encoding="utf-8") - # Inside the committed entries' window (they take force 2026-09-22). - green_certification_inputs["exclusions_evaluated_on"] = date(2026, 10, 1) with pytest.raises(UKReleaseCertificationError, match="not the committed register"): compose_uk_release_certification(**green_certification_inputs) - receipt["incumbent_unresolvable_pruned"]["reviewed_register"]["sha256"] = ( - uk_incumbent_unresolvable_register_digest()["sha256"] + receipt["reference_unresolvable_pruned"]["reviewed_register"]["sha256"] = ( + candidate_score.uk_reference_unresolvable_register_digest()["sha256"] ) path.write_text(json.dumps(receipt), encoding="utf-8") certification = compose_uk_release_certification(**green_certification_inputs) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index 9cfbe1551d..8543b7d9bf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -40,6 +40,10 @@ require_committed_uk_chronicle_feed_pin, ) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.cut_reference import ( + UKCutReference, + write_uk_cut_reference, +) from microcosm.build.uk_runtime.national_calibration import ( CalibrationFrameAdapter, national_calibration_mass_reason, @@ -1365,15 +1369,18 @@ def test_uk_national_role_records_an_unpinned_feed_override( assert operations["configuration"]["allow_unpinned_feed"] is True -# --- the end-of-build incumbent evaluation (microcosm#823 E2) ----------------- +# --- the end-of-build comparison with the reference (microcosm#823 E2, #1095) -- + +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +REFERENCE_LABEL = f"{UK_NATIONAL_RELEASE_ID}@{REFERENCE_BUILD}" def _fake_evaluator(receipt_of, calls: list[dict]): - """A stand-in for microcosm.build.uk_runtime.candidate_score (#967).""" + """A stand-in for microcosm.build.uk_runtime.candidate_score.""" class Module: @staticmethod - def evaluate_uk_candidate_against_incumbent(**kwargs): + def evaluate_uk_candidate_against_reference(**kwargs): calls.append(kwargs) return receipt_of(kwargs) @@ -1383,7 +1390,7 @@ def uk_default_measure_resolver_factory(scratch_dir, year): @staticmethod def pruned_warning(score): - pruned = score.get("incumbent_unresolvable_pruned") or {} + pruned = score.get("reference_unresolvable_pruned") or {} if not pruned.get("n_pruned"): return None return "warning: pruned from BOTH arms: " + ", ".join(pruned["measures"]) @@ -1391,37 +1398,44 @@ def pruned_warning(score): return Module -def _receipt(kwargs, *, verdict="passed", pruned_measures=()): +def _receipt(kwargs, *, pruned_measures=()): n_pruned = len(pruned_measures) + targets = {"candidate_lower_count": 1, "reference_lower_count": 0, "equal_count": 0} return { "artifacts": { "candidate": {"sha256": kwargs["candidate_sha256"]}, - "incumbent": { - "sha256": kwargs["incumbent_sha256"], - "label": kwargs["incumbent_label"], + "reference": { + "sha256": kwargs["reference_sha256"], + "label": kwargs["reference_label"], }, }, "candidate_full_loss": 0.01, - "incumbent_full_loss": 0.2, - "candidate_target_wins": 1, - "incumbent_target_wins": 0, + "reference_full_loss": 0.2, "register": {"n_specs": 1}, - # Record arrays the reviewed telemetry artifact must not carry. - "target_drift": [ - { - "target": "dwp.uc.households@2025", - "family": "dwp_universal_credit", - "candidate_relative_error": 0.01, - "incumbent_relative_error": 0.2, - "winner": "candidate", - } - ], - "signed_asymmetries": [{"id": "incumbent_own_registry", "description": "…"}], + # A record array the reviewed telemetry artifact must not carry. + "head_to_head": { + "schema_version": 1, + "loss": {"candidate": 0.01, "reference": 0.2}, + "targets": { + **targets, + "compared": [ + { + "name": "dwp.uc.households@2025", + "family": "dwp_universal_credit", + "candidate_relative_error": 0.01, + "reference_relative_error": 0.2, + "lower_absolute_relative_error": "candidate", + } + ], + }, + "by_family": {"dwp_universal_credit": targets}, + }, + "signed_asymmetries": [{"id": "reference_own_registry", "description": "…"}], "measure_resolution": { "candidate": {"rounds": [{"round": 0}]}, - "incumbent": None, + "reference": None, }, - "incumbent_unresolvable_pruned": { + "reference_unresolvable_pruned": { "n_pruned": n_pruned, "n_scored": 1, "n_surface": 1 + n_pruned, @@ -1430,63 +1444,78 @@ def _receipt(kwargs, *, verdict="passed", pruned_measures=()): "families": {"dwp_universal_credit": n_pruned} if n_pruned else {}, "note": "pruned", }, + "reference": kwargs["reference_record"], "evaluation": { - "schema_version": 1, - "rule": "rule 1", + "schema_version": 2, + "rule": "microcosm#1095", + "reference": kwargs["reference_label"], "scored_surface": { "n_scored": 1, "n_pruned": n_pruned, "n_surface": 1 + n_pruned, }, - "rule_1": { - "passed": verdict == "passed", - "candidate_full_loss": 0.01, - "incumbent_full_loss": 0.2, - "candidate_target_wins": 1, - "incumbent_target_wins": 0, - }, - "verdict": verdict, + "losses": {"candidate_full_loss": 0.01, "reference_full_loss": 0.2}, + "targets": targets, + "no_threshold_applied": True, }, } -def _incumbent_argv(input_h5: Path, tmp_path: Path, *extra: str) -> list[str]: - incumbent = tmp_path / "incumbent.h5" - shutil.copyfile(input_h5, incumbent) +def _reference_argv(input_h5: Path, tmp_path: Path, *extra: str) -> list[str]: + """A reference file and the record a cut's fetch writes for it.""" + + reference = tmp_path / "reference.h5" + shutil.copyfile(input_h5, reference) + record = UKCutReference( + release=UK_NATIONAL_RELEASE_ID, + build=REFERENCE_BUILD, + repo_id="policyengine/populace-uk-private", + path_in_repo=f"releases/{UK_NATIONAL_RELEASE_ID}/{REFERENCE_BUILD}/x.h5", + filename=reference.name, + sha256=_sha(reference), + size_bytes=reference.stat().st_size, + built_with_model_package={"name": "policyengine-uk", "version": "2.174.1"}, + built_with_core_package={"name": "policyengine-core", "version": "3.33.3"}, + evidence={ + "key": "release_certification", + "filename": "release_certification.json", + "sha256": "1" * 64, + }, + resolved_at="2026-11-10T12:00:00Z", + ) + record_path = write_uk_cut_reference(tmp_path / "reference.json", record) return [ - "--incumbent-h5", - str(incumbent), - "--incumbent-sha256", - _sha(incumbent), - "--incumbent-label", - "efrs_fixture", + "--reference-h5", + str(reference), + "--reference-record", + str(record_path), *extra, ] -def test_uk_dense_role_refuses_the_incumbent_flags(tmp_path) -> None: +def test_uk_dense_role_refuses_the_national_reference_flags(tmp_path) -> None: builder = candidate._load_builder_module() args = argparse.Namespace( target_loss_cap=None, allow_unpinned_feed=False, target_weight_rule="grain_equal", - incumbent_h5=tmp_path / "incumbent.h5", - incumbent_sha256=None, + reference_h5=tmp_path / "reference.h5", + reference_record=None, ) - with pytest.raises(ValueError, match="--incumbent-h5/--incumbent-sha256"): + with pytest.raises(ValueError, match="--reference-h5/--reference-record"): builder._refuse_national_role_arguments( args, builder.uk_rowwise_posture("dense") ) -def test_uk_national_role_requires_the_incumbent_pair(monkeypatch, tmp_path): +def test_uk_national_role_requires_the_reference_pair(monkeypatch, tmp_path): pytest.importorskip("tables") builder = candidate._load_builder_module() input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) out = tmp_path / "out" assert ( builder.main( - _argv(input_h5, out, "--no-staging", "--incumbent-h5", str(input_h5)) + _argv(input_h5, out, "--no-staging", "--reference-h5", str(input_h5)) ) == 1 ) @@ -1495,17 +1524,31 @@ def test_uk_national_role_requires_the_incumbent_pair(monkeypatch, tmp_path): assert not (out / "microcosm_uk_2024_25.h5").exists() -def test_uk_national_role_refuses_the_incumbent_without_the_scorer() -> None: +def test_uk_national_role_refuses_a_reference_the_record_does_not_name( + monkeypatch, tmp_path +): + pytest.importorskip("tables") + builder = candidate._load_builder_module() + input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) + argv = _reference_argv(input_h5, tmp_path) + (tmp_path / "reference.h5").write_bytes(b"another file") + out = tmp_path / "out" + + assert builder.main(_argv(input_h5, out, "--no-staging", *argv)) == 1 + failure = json.loads((out / "failure.json").read_text()) + assert "is" in failure["message"] and "bytes" in failure["message"] + assert not (out / "microcosm_uk_2024_25.h5").exists() + + +def test_uk_national_role_refuses_the_reference_without_the_scorer() -> None: def missing(name: str): raise ImportError(name) - with pytest.raises(ValueError, match="microcosm#967"): + with pytest.raises(ValueError, match="head-to-head scorer"): national_role._load_candidate_evaluator(importer=missing) -def test_uk_national_role_evaluates_against_the_incumbent( - monkeypatch, tmp_path, capsys -): +def test_uk_national_role_compares_with_the_reference(monkeypatch, tmp_path, capsys): pytest.importorskip("tables") builder = candidate._load_builder_module() input_h5, registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) @@ -1529,7 +1572,7 @@ def test_uk_national_role_evaluates_against_the_incumbent( "--no-staging", "--epochs", "5", - *_incumbent_argv(input_h5, tmp_path), + *_reference_argv(input_h5, tmp_path), ) ) == 0 @@ -1538,22 +1581,27 @@ def test_uk_national_role_evaluates_against_the_incumbent( call = calls[0] assert call["candidate_h5"] == out / "microcosm_uk_2024_25.h5" assert call["candidate_sha256"] == _sha(out / "microcosm_uk_2024_25.h5") - assert call["incumbent_h5"] == tmp_path / "incumbent.h5" - assert call["incumbent_sha256"] == _sha(tmp_path / "incumbent.h5") - assert call["incumbent_label"] == "efrs_fixture" + assert call["reference_h5"] == tmp_path / "reference.h5" + assert call["reference_sha256"] == _sha(tmp_path / "reference.h5") + assert call["reference_label"] == REFERENCE_LABEL + assert call["reference_record"]["build"] == REFERENCE_BUILD # The registries the scorer receives are the ones the graph solved and # froze, by content hash. assert call["target_registry"].version == registry.version assert call["band_edge_registry"].version == registry.version assert call["calibration_year"] == 2025 - receipt_path = out / "score_vs_incumbent.json" + receipt_path = out / "score_vs_reference.json" receipt = json.loads(receipt_path.read_text()) - assert receipt["evaluation"]["verdict"] == "passed" + assert receipt["evaluation"]["reference"] == REFERENCE_LABEL manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) evaluation = manifest["evaluation"] assert evaluation["status"] == "completed" - assert evaluation["verdict"] == "passed" - assert evaluation["rule_1"]["passed"] is True + assert evaluation["reference"] == REFERENCE_LABEL + assert "verdict" not in evaluation + assert evaluation["losses"] == { + "candidate_full_loss": 0.01, + "reference_full_loss": 0.2, + } assert evaluation["scored_surface"] == { "n_scored": 1, "n_pruned": 1, @@ -1562,8 +1610,8 @@ def test_uk_national_role_evaluates_against_the_incumbent( assert evaluation["pruned_measures"] == ["benunit.uc_calibration_child_count"] assert evaluation["pruned_families"] == {"dwp_universal_credit": 1} assert evaluation["receipt"]["sha256"] == _sha(receipt_path) - assert evaluation["incumbent"]["label"] == "efrs_fixture" - assert evaluation["incumbent"]["sha256"] == _sha(tmp_path / "incumbent.h5") + assert evaluation["reference_input"]["label"] == REFERENCE_LABEL + assert evaluation["reference_input"]["sha256"] == _sha(tmp_path / "reference.h5") assert manifest["outputs"]["score_receipt"]["sha256"] == _sha(receipt_path) # The absent measure is named loudly on stderr, never hidden. assert "benunit.uc_calibration_child_count" in capsys.readouterr().err @@ -1577,7 +1625,7 @@ def test_uk_national_role_records_an_evaluation_error_without_failing( input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) def explode(kwargs): - raise RuntimeError("the incumbent frame refused to load") + raise RuntimeError("the reference frame refused to load") module = _fake_evaluator(explode, []) monkeypatch.setattr( @@ -1593,7 +1641,7 @@ def explode(kwargs): "--no-staging", "--epochs", "5", - *_incumbent_argv(input_h5, tmp_path), + *_reference_argv(input_h5, tmp_path), ) ) == 0 @@ -1604,8 +1652,8 @@ def explode(kwargs): assert "refused to load" in manifest["evaluation"]["error"] assert manifest["evaluation"]["receipt"] is None assert "score_receipt" not in manifest["outputs"] - assert not (out / "score_vs_incumbent.json").exists() - assert "the incumbent evaluation failed" in capsys.readouterr().err + assert not (out / "score_vs_reference.json").exists() + assert "the comparison with the reference failed" in capsys.readouterr().err def test_uk_national_role_evaluates_after_staging_the_bundle( @@ -1629,7 +1677,7 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( "--staging-local-only", "--epochs", "5", - *_incumbent_argv(input_h5, tmp_path), + *_reference_argv(input_h5, tmp_path), ) ) == 0 @@ -1638,14 +1686,14 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) run_id = manifest["build_id"] assert manifest["evaluation"]["status"] == "completed" - assert manifest["evaluation"]["verdict"] == "passed" + assert manifest["evaluation"]["reference"] == REFERENCE_LABEL assert manifest["outputs"]["score_receipt"]["sha256"] == _sha( - out / "score_vs_incumbent.json" + out / "score_vs_reference.json" ) # The receipt is listed in the local sums beside the record and manifest. _sums_verify(out) assert any( - line.endswith(" score_vs_incumbent.json") + line.endswith(" score_vs_reference.json") for line in (out / "sha256sums.txt").read_text().splitlines() ) bundle = validate_v2_bundle(out / "staging", run_id) @@ -1653,28 +1701,30 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( events = [ (event["stage_id"], event["status"]) for event in bundle["events"] - if event["stage_id"] == "incumbent_evaluation" + if event["stage_id"] == "reference_evaluation" ] assert events == [ - ("incumbent_evaluation", "started"), - ("incumbent_evaluation", "completed"), + ("reference_evaluation", "started"), + ("reference_evaluation", "completed"), ] # The evaluation ran after the bundle was staged, before completion. stage_ids = [event["stage_id"] for event in bundle["events"]] - assert stage_ids.index("dataset_staging") < stage_ids.index("incumbent_evaluation") - assert stage_ids.index("incumbent_evaluation") < stage_ids.index("complete") + assert stage_ids.index("dataset_staging") < stage_ids.index("reference_evaluation") + assert stage_ids.index("reference_evaluation") < stage_ids.index("complete") artifacts = out / "staging" / "runs" / run_id / "artifacts" - staged_receipt = json.loads((artifacts / "score_vs_incumbent.json").read_text()) - assert staged_receipt["evaluation"]["verdict"] == "passed" - # The staged copy keeps the verdict, the pruned block and the aggregates - # and drops the record arrays the reviewed-artifact policy refuses; the - # full receipt is beside the outputs. - assert "incumbent_unresolvable_pruned" in staged_receipt - assert staged_receipt["candidate_target_wins"] == 1 - for key in ("target_drift", "signed_asymmetries", "measure_resolution"): + staged_receipt = json.loads((artifacts / "score_vs_reference.json").read_text()) + assert staged_receipt["evaluation"]["reference"] == REFERENCE_LABEL + # The staged copy keeps the summary, the pruned block and the counts and + # drops the record arrays the reviewed-artifact policy refuses; the full + # receipt is beside the outputs. + assert "reference_unresolvable_pruned" in staged_receipt + assert staged_receipt["head_to_head"]["targets"]["candidate_lower_count"] == 1 + assert "compared" not in staged_receipt["head_to_head"]["targets"] + for key in ("signed_asymmetries", "measure_resolution"): assert key not in staged_receipt - full_receipt = json.loads((out / "score_vs_incumbent.json").read_text()) - assert full_receipt["target_drift"][0]["winner"] == "candidate" + full_receipt = json.loads((out / "score_vs_reference.json").read_text()) + compared = full_receipt["head_to_head"]["targets"]["compared"] + assert compared[0]["lower_absolute_relative_error"] == "candidate" assert calls[0]["candidate_sha256"] == manifest["outputs"]["dataset"]["sha256"] # The completion marker binds the final bytes: the build record gained # the delivery summary and the manifest its receipts after publication, @@ -1690,7 +1740,7 @@ def test_uk_national_role_evaluates_after_staging_the_bundle( assert completion["dataset"]["sha256"] == _sha(out / "microcosm_uk_2024_25.h5") -def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys): +def test_uk_national_dry_run_records_the_reference(monkeypatch, tmp_path, capsys): pytest.importorskip("tables") builder = candidate._load_builder_module() input_h5, _registry, _artifact, _pin = _national_inputs(monkeypatch, tmp_path) @@ -1706,7 +1756,7 @@ def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys tmp_path / "plan", "--dry-run", "--no-staging", - *_incumbent_argv(input_h5, tmp_path), + *_reference_argv(input_h5, tmp_path), ) ) == 0 @@ -1714,8 +1764,9 @@ def test_uk_national_dry_run_records_the_incumbent(monkeypatch, tmp_path, capsys plan = json.loads(capsys.readouterr().out) assert plan["dry_run"] is True - assert plan["incumbent"]["label"] == "efrs_fixture" - assert plan["incumbent"]["sha256"] == _sha(tmp_path / "incumbent.h5") + assert plan["reference"]["label"] == REFERENCE_LABEL + assert plan["reference"]["sha256"] == _sha(tmp_path / "reference.h5") + assert "record" not in plan["reference"] assert not (tmp_path / "plan").exists() @@ -1725,13 +1776,14 @@ def test_uk_national_role_loads_the_real_scorer_by_its_public_name() -> None: module = national_role._load_candidate_evaluator() assert module.__name__ == "microcosm.build.uk_runtime.candidate_score" assert callable(module.uk_default_measure_resolver_factory) - assert callable(module.evaluate_uk_candidate_against_incumbent) + assert callable(module.evaluate_uk_candidate_against_reference) def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: - """The staged copy drops the three per-target arrays and keeps the - pruned block whole, including a populated pruned_targets mapping (the - block a real run fills with 120 rows; Vahid's #965 note 2).""" + """The staged copy drops the per-target comparisons and the two other + record arrays and keeps the pruned block whole, including a populated + pruned_targets mapping (the block a real run fills with many rows; + Vahid's #965 note 2).""" pruned_targets = { f"dwp/uc/family_{i}": { "name": f"dwp/uc/family_{i}", @@ -1744,10 +1796,18 @@ def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: } score = { "artifacts": {"candidate": {"sha256": "1" * 64}}, - "target_drift": [{"target": "a@0", "error": 0.1}], + "head_to_head": { + "targets": { + "candidate_lower_count": 1, + "reference_lower_count": 0, + "equal_count": 0, + "compared": [{"name": "a@0", "lower_absolute_relative_error": "x"}], + }, + "by_family": {}, + }, "signed_asymmetries": [{"id": "x"}], "measure_resolution": {"candidate": {"rounds": []}}, - "incumbent_unresolvable_pruned": { + "reference_unresolvable_pruned": { "n_pruned": 3, "n_scored": 1, "n_surface": 4, @@ -1757,15 +1817,21 @@ def test_uk_score_receipt_telemetry_summary_keeps_the_pruned_block() -> None: "reviewed_register": {"resource": "r.json", "sha256": "2" * 64}, "note": "pruned", }, - "evaluation": {"verdict": "passed"}, + "evaluation": {"reference": REFERENCE_LABEL}, } summary = national_role._score_receipt_telemetry_summary(score) assert set(summary) == { "artifacts", - "incumbent_unresolvable_pruned", + "head_to_head", + "reference_unresolvable_pruned", "evaluation", } - assert summary["incumbent_unresolvable_pruned"]["pruned_targets"] == pruned_targets + assert summary["head_to_head"]["targets"] == { + "candidate_lower_count": 1, + "reference_lower_count": 0, + "equal_count": 0, + } + assert summary["reference_unresolvable_pruned"]["pruned_targets"] == pruned_targets # No list of mappings survives (the staging content policy's record # array refusal); the pruned rows are a mapping keyed by target name. diff --git a/packages/microcosm-build/tests/uk_certification_fixtures.py b/packages/microcosm-build/tests/uk_certification_fixtures.py index 0e7d7a1d9f..3a8093fdf5 100644 --- a/packages/microcosm-build/tests/uk_certification_fixtures.py +++ b/packages/microcosm-build/tests/uk_certification_fixtures.py @@ -120,20 +120,67 @@ def sha256(path: Path) -> str: GREEN_PARENT_SPINE_SHA256 = "a" * 64 +#: The reference the green fixture's cut recorded (microcosm#1095). +GREEN_REFERENCE_RECORD = { + "kind": "uk_cut_reference", + "schema_version": 1, + "release": "microcosm-uk-2024-25-national", + "build": "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e", + "repo_id": "policyengine/populace-uk-private", + "path_in_repo": ( + "releases/microcosm-uk-2024-25-national/" + "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e/microcosm_uk_2024_25.h5" + ), + "filename": "microcosm_uk_2024_25.h5", + "sha256": "9" * 64, + "size_bytes": 1, + "built_with_model_package": {"name": "policyengine-uk", "version": "2.174.1"}, + "built_with_core_package": {"name": "policyengine-core", "version": "3.33.3"}, + "evidence": { + "key": "release_certification", + "filename": "release_certification.json", + "sha256": "8" * 64, + }, + "resolved_at": "2026-11-10T12:00:00Z", +} +GREEN_REFERENCE_LABEL = ( + f"{GREEN_REFERENCE_RECORD['release']}@{GREEN_REFERENCE_RECORD['build']}" +) + + def green_score_receipt(candidate_sha: str) -> dict: """A score receipt the certifier accepts: measured on the candidate's - bytes, scored on a closed common surface, rule 1 passed.""" + bytes against the cut's reference, on a closed common surface.""" + targets = { + "candidate_lower_count": 293, + "reference_lower_count": 12, + "equal_count": 2, + } return { "artifacts": { "candidate": {"sha256": candidate_sha, "size_bytes": 15}, - "incumbent": {"sha256": "9" * 64, "size_bytes": 1}, + "reference": { + "label": GREEN_REFERENCE_LABEL, + "sha256": GREEN_REFERENCE_RECORD["sha256"], + "size_bytes": 1, + }, }, "candidate_full_loss": 0.0096, - "incumbent_full_loss": 0.211, - "candidate_target_wins": 293, - "incumbent_target_wins": 12, - "incumbent_unresolvable_pruned": { + "reference_full_loss": 0.0101, + "head_to_head": { + "schema_version": 1, + "decision": "owner_decides", + "no_threshold_applied": True, + "loss": { + "candidate": 0.0096, + "reference": 0.0101, + "candidate_minus_reference": 0.0096 - 0.0101, + }, + "targets": {**targets, "compared": []}, + "by_family": {}, + }, + "reference_unresolvable_pruned": { "n_pruned": 0, "n_scored": 307, "n_surface": 307, @@ -141,24 +188,22 @@ def green_score_receipt(candidate_sha: str) -> dict: "measures": [], "families": {}, "reviewed_register": { - "resource": "incumbent_unresolvable_measures.json", + "resource": "reference_unresolvable_measures.json", "sha256": None, + "reference": GREEN_REFERENCE_LABEL, "entries_used": {}, }, "note": "nothing pruned", }, + "reference": dict(GREEN_REFERENCE_RECORD), "evaluation": { - "schema_version": 1, - "rule": "microcosm#578 rule 1 on the common surface", + "schema_version": 2, + "rule": "microcosm#1095: the candidate and the current build", + "reference": GREEN_REFERENCE_LABEL, "scored_surface": {"n_scored": 307, "n_pruned": 0, "n_surface": 307}, - "rule_1": { - "passed": True, - "candidate_full_loss": 0.0096, - "incumbent_full_loss": 0.211, - "candidate_target_wins": 293, - "incumbent_target_wins": 12, - }, - "verdict": "passed", + "losses": {"candidate_full_loss": 0.0096, "reference_full_loss": 0.0101}, + "targets": targets, + "no_threshold_applied": True, }, } @@ -242,7 +287,7 @@ def green_certification_inputs(tmp_path: Path): "terminal_gate_json": {"sha256": seam_report_sha}, }, } - score_receipt = tmp_path / "score_vs_enhanced_frs.json" + score_receipt = tmp_path / "score_vs_reference.json" score_receipt.write_text( json.dumps(green_score_receipt(candidate_sha)), encoding="utf-8" ) @@ -258,6 +303,7 @@ def green_certification_inputs(tmp_path: Path): "spine_sidecar": sidecar, "build_record": build_record, "score_receipt_path": score_receipt, + "reference_record": dict(GREEN_REFERENCE_RECORD), "exclusions_evaluated_on": date(2026, 8, 27), "certification_path": tmp_path / "release_certification.json", } diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index d5016c34ce..451577a318 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -998,8 +998,16 @@ def _uk_line_cut_tag_re(release_id: str) -> re.Pattern[str] | None: "calibration_seam": "terminal_gates.json", "release_cut": "release_cut_gates.json", } -_UK_CERTIFICATION_SCORE_RECEIPT_FILE = "score_vs_enhanced_frs.json" -_UK_RELEASE_CERTIFICATION_SCHEMA_VERSION = 1 +#: The head-to-head receipt against the cut's reference, the current +#: national build (microcosm#1095); certification schema 1 named +#: ``score_vs_enhanced_frs.json``, a strict score against the enhanced FRS. +#: Only new builds are validated (the assemblers call this contract), at the +#: spec they are assembled under, so schema 1 has no path here. +_UK_CERTIFICATION_SCORE_RECEIPT_FILE = "score_vs_reference.json" +_UK_RELEASE_CERTIFICATION_SCHEMA_VERSION = 2 +_UK_CERTIFICATION_REFERENCE_FIELDS = frozenset( + {"release", "build", "filename", "sha256", "built_with_model_package"} +) _UK_RELEASE_CERTIFICATION_KIND = "uk_release_certification" _UK_CERTIFICATION_SHARED_GATE_IDS = frozenset({"uk_aggregate_admin"}) _UK_CERTIFICATION_EXCLUDED_GATE_IDS = frozenset( @@ -1156,6 +1164,8 @@ def _uk_line_cut_tag_re(release_id: str) -> re.Pattern[str] | None: # The spine whose stage receipts the release cut measured, bound by # the certifier to the parent the calibration recorded. "parent_spine", + # The current national build the cut was compared with. + "reference", "parts", "spec", "doctrine", @@ -3499,6 +3509,35 @@ def _check_uk_release_certification( f"{file} release_id {certification.get('release_id')!r} does not " f"match the release under validation ({release_id!r})." ) + reference = certification.get("reference") + if ( + not isinstance(reference, Mapping) + or set(reference) != _UK_CERTIFICATION_REFERENCE_FIELDS + ): + failures.append( + f"{file} reference must name the build the cut was compared with: " + f"exactly {sorted(_UK_CERTIFICATION_REFERENCE_FIELDS)}." + ) + else: + package = reference.get("built_with_model_package") + if ( + reference.get("release") != _UK_NATIONAL_RELEASE_ID + or not isinstance(reference.get("build"), str) + or not reference.get("build") + or not isinstance(reference.get("filename"), str) + or not reference.get("filename") + or not isinstance(reference.get("sha256"), str) + or not _SHA256_RE.fullmatch(reference.get("sha256")) + or not isinstance(package, Mapping) + or set(package) != {"name", "version"} + or package.get("name") != "policyengine-uk" + or not isinstance(package.get("version"), str) + ): + failures.append( + f"{file} reference must be a build of {_UK_NATIONAL_RELEASE_ID} " + "with its file name, sha256 digest and built-with " + "policyengine-uk version." + ) parent_spine = certification.get("parent_spine") parent_sha = ( parent_spine.get("sha256") if isinstance(parent_spine, Mapping) else None diff --git a/packages/microcosm-data/tests/engine_free/shared/test_contract.py b/packages/microcosm-data/tests/engine_free/shared/test_contract.py index 163a63decb..80327cb422 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_contract.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_contract.py @@ -1808,9 +1808,7 @@ def _write_uk_national_release_dir(tmp_path: Path) -> Path: (national / "release_cut_gates.json").write_text( json.dumps({"fixture": "release_cut"}) ) - (national / "score_vs_enhanced_frs.json").write_text( - json.dumps({"fixture": "score"}) - ) + (national / "score_vs_reference.json").write_text(json.dumps({"fixture": "score"})) certification = _green_uk_certification( TEST_UK_TERMINAL_GATE_SIGNING_KEY_BYTES, release_id=UK_NATIONAL_RELEASE_ID, @@ -1820,7 +1818,7 @@ def _write_uk_national_release_dir(tmp_path: Path) -> Path: "calibration_seam": _sha256(national / "terminal_gates.json"), "release_cut": _sha256(national / "release_cut_gates.json"), }, - score_receipt_sha256=_sha256(national / "score_vs_enhanced_frs.json"), + score_receipt_sha256=_sha256(national / "score_vs_reference.json"), ) certification_path = national / "release_certification.json" certification_path.write_text(json.dumps(certification)) @@ -6060,7 +6058,7 @@ def _green_uk_certification( "statuses": {"passed": len(scope)}, } certification = { - "schema_version": 1, + "schema_version": 2, "kind": "uk_release_certification", "country": "uk", "release_id": release_id, @@ -6071,6 +6069,16 @@ def _green_uk_certification( "size_bytes": 1, }, "parent_spine": {"sha256": "a" * 64}, + "reference": { + "release": "microcosm-uk-2024-25-national", + "build": "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e", + "filename": "microcosm_uk_2024_25.h5", + "sha256": "9" * 64, + "built_with_model_package": { + "name": "policyengine-uk", + "version": "2.174.1", + }, + }, "parts": parts, "spec": { "gates_manifest_sha256": contract._UK_GATE_BATTERY_GATES_MANIFEST_SHA256, @@ -6086,7 +6094,7 @@ def _green_uk_certification( "doctrine": {"payload": {"epochs": 1500}, "overrides": {}}, "diagnostics_sha256": diagnostics_sha256, "score_receipt": { - "filename": "score_vs_enhanced_frs.json", + "filename": "score_vs_reference.json", "sha256": score_receipt_sha256, }, "exclusions_evaluated_on": "2026-08-27", @@ -6195,6 +6203,32 @@ def test_uk_release_certification_refusals(monkeypatch) -> None: for line in _certification_failures(certification, monkeypatch, key) ) + # Schema 2 names the build the cut was compared with (microcosm#1095); + # a schema-1 certification against the enhanced FRS no longer validates. + certification = _green_uk_certification(key) + del certification["reference"] + assert any( + "exactly the certification fields" in line + for line in _certification_failures(certification, monkeypatch, key) + ) + for broken in ( + {"release": "microcosm-uk-2024-25-dense"}, + {"sha256": "not-a-digest"}, + {"built_with_model_package": {"name": "policyengine-us", "version": "1"}}, + ): + certification = _green_uk_certification(key) + certification["reference"].update(broken) + assert any( + "reference must be a build of" in line + for line in _certification_failures(certification, monkeypatch, key) + ), broken + certification = _green_uk_certification(key) + certification["schema_version"] = 1 + assert any( + "schema_version must be the integer 2" in line + for line in _certification_failures(certification, monkeypatch, key) + ) + def test_national_line_artifacts_require_the_certification(tmp_path) -> None: # A release that ships any national-line gate part without the composed diff --git a/tools/assemble_uk_release_dir.py b/tools/assemble_uk_release_dir.py index d4ab612d2c..863cec7184 100644 --- a/tools/assemble_uk_release_dir.py +++ b/tools/assemble_uk_release_dir.py @@ -464,9 +464,10 @@ def _stage_and_finalize( "release_cut_gates.json", ), "spine_gates": (spine_report_path, "spine_gates.json"), - "score_vs_enhanced_frs": ( + # The head-to-head with the reference (microcosm#1095). + "score_vs_reference": ( args.score_receipt, - "score_vs_enhanced_frs.json", + "score_vs_reference.json", ), } @@ -553,8 +554,8 @@ def artifact(key: str, kind: str, path: str, sha256: str) -> dict[str, str]: "spine report", ), ( - "score_vs_enhanced_frs", - evidence["score_vs_enhanced_frs"], + "score_vs_reference", + evidence["score_vs_reference"], "score receipt", ), ) diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 8f8c18682d..7c5042316a 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -12,9 +12,12 @@ only from the certification this driver writes. The battery always runs at release-candidate strictness: evidence_absent -gaps block. The rule-1 score receipt is cross-pinned into the certification -(the audit's third carried defect), so the score is signed run evidence -rather than a null slot. +gaps block. The head-to-head score receipt is cross-pinned into the +certification (the audit's third carried defect), so the comparison is signed +run evidence rather than a null slot. The certification names the reference +the cut was compared with, the current national build the cut recorded +(``--reference-record``, microcosm#1095), and refuses a receipt measured +against any other. """ from __future__ import annotations @@ -372,6 +375,7 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: spine_sidecar=sidecar, build_record=build_record, score_receipt_path=args.score_receipt, + reference_record=json.loads(args.reference_record.read_text()), exclusions_evaluated_on=evaluated_on, certification_path=args.certification_json, ) @@ -408,6 +412,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) parser.add_argument("--input-mass-reference", required=True, type=Path) parser.add_argument("--score-receipt", required=True, type=Path) + parser.add_argument( + "--reference-record", + required=True, + type=Path, + help="The cut's record of the reference it was compared with.", + ) parser.add_argument("--release-id", required=True) parser.add_argument("--release-cut-gate-json", type=Path) parser.add_argument("--certification-json", type=Path) @@ -429,6 +439,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: "--release-cut-gate-json": args.release_cut_gate_json, "--certification-json": args.certification_json, "--score-receipt": args.score_receipt, + "--reference-record": args.reference_record, } resolved: dict[Path, str] = {} for flag, path in distinct.items(): diff --git a/tools/extract_uk_local_incumbent_surface.py b/tools/extract_uk_local_incumbent_surface.py index 243ca6a82c..5adb5cf315 100644 --- a/tools/extract_uk_local_incumbent_surface.py +++ b/tools/extract_uk_local_incumbent_surface.py @@ -216,10 +216,9 @@ def extract_incumbent_surface( f"{incumbent_h5}: the local UC child bands read " "benunit.uc_calibration_child_count, and this incumbent lacks " f"its relationship inputs ({error.args[0]}). The enhanced FRS " - "is known not to carry that measure " - "(uk/incumbent_unresolvable_measures.json); the local extractor " - "does not prune listed measures, so use a Microcosm release as " - "the incumbent." + "is known not to carry that measure, and the local extractor " + "does not prune unresolvable measures, so use a Microcosm " + "release as the incumbent." ) from error metrics = metric_frames[0].copy() for frame in metric_frames[1:]: diff --git a/tools/resolve_uk_reference.py b/tools/resolve_uk_reference.py index 4738e9a4b6..2f114dea09 100644 --- a/tools/resolve_uk_reference.py +++ b/tools/resolve_uk_reference.py @@ -24,21 +24,17 @@ import argparse import json -import shutil import sys from collections.abc import Sequence from pathlib import Path from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_RECORD_NAME, UK_CUT_REFERENCE_RELEASES, check_uk_cut_reference, - resolve_uk_cut_reference, - verify_uk_cut_reference_file, - write_uk_cut_reference, + fetch_uk_cut_reference, ) -REFERENCE_RECORD_NAME = "reference.json" - def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) @@ -54,23 +50,12 @@ def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: def fetch_reference(release: str, output_dir: Path, *, hub_download=None) -> dict: """Download the current build of ``release`` into ``output_dir``.""" - record_path = output_dir / REFERENCE_RECORD_NAME - if record_path.exists(): - raise ValueError( - f"{output_dir} already holds a reference ({record_path}); a run keeps " - "the reference it started with. Use a new directory to resolve again." - ) - source, reference = resolve_uk_cut_reference(release, hub_download=hub_download) - output_dir.mkdir(parents=True, exist_ok=True) - destination = output_dir / reference.filename - shutil.copyfile(source, destination) - verify_uk_cut_reference_file(reference, destination) - write_uk_cut_reference(record_path, reference) + reference = fetch_uk_cut_reference(release, output_dir, hub_download=hub_download) return { "reference": reference.label, "sha256": reference.sha256, - "dataset": str(destination), - "record": str(record_path), + "dataset": str(output_dir / reference.filename), + "record": str(output_dir / UK_CUT_REFERENCE_RECORD_NAME), } diff --git a/tools/run_uk_cut.py b/tools/run_uk_cut.py index f66e93c031..7787640f4a 100644 --- a/tools/run_uk_cut.py +++ b/tools/run_uk_cut.py @@ -19,6 +19,14 @@ or a release directory. Re-run ``all`` after fixing the cause; a failed national run is moved aside only with ``--retry-national``. +The cut checks and certifies its candidate against the current build of the +national line, as the Hub's ``releases.json`` names it (microcosm#1095). The +``reference`` step resolves it once per cut, into ``/reference/`` with +its record; every later step reads that record, so a promotion while the cut +runs does not change what it is compared with. ``--re-resolve-reference`` +sets the cut's reference aside and resolves the current build again; the +score and the certification then run again against it. + Outward-facing steps are opt-in. ``--publish`` publishes for inspection only, always with ``--no-latest --tag-only``, so the repository's main branch and its ``latest-*.json`` pointers never move. Promotion and the @@ -54,6 +62,15 @@ from pathlib import Path from typing import Any, Protocol +from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_RECORD_NAME, + UKCutReference, + check_uk_cut_reference, + fetch_uk_cut_reference, + read_uk_cut_reference, + verify_uk_cut_reference_file, +) + REPO_ROOT = Path(__file__).resolve().parents[1] UK_DIR = REPO_ROOT / "packages/microcosm-build/src/microcosm/build/uk" CUT_INPUTS = REPO_ROOT / "tools/uk_cut_inputs.json" @@ -166,21 +183,10 @@ def committed_source_pins(uk_dir: Path = UK_DIR) -> dict[str, dict[str, Any]]: return pins -def _gate_reference_identity(uk_dir: Path) -> Mapping[str, str]: - gates = json.loads((uk_dir / "gates.json").read_text()) - entries = gates["gates"] if isinstance(gates, Mapping) else gates - for entry in entries: - if entry.get("gate") == "input_mass_parity": - params = entry["parameters"] - return params["reference_registry"][params["reference"]]["identity"] - raise CutError("gates.json declares no input_mass_parity reference identity.") - - #: The ``*_pin`` selectors a source may carry, each naming the committed pin #: a field is read from. Any other ``*_pin`` is refused, never ignored. PIN_SELECTORS: Mapping[str, frozenset[str]] = { "archive_sha256_pin": frozenset({"frs_release"}), - "revision_pin": frozenset({"gates:input_mass_parity"}), "path_in_repo_pin": frozenset({"chronicle_feed"}), } @@ -208,7 +214,6 @@ def load_input_sources( stage_pins = committed_source_pins(uk_dir) feed = json.loads((uk_dir / "chronicle_feed.json").read_text()) frs_release = json.loads((uk_dir / "frs_release.json").read_text()) - reference = _gate_reference_identity(uk_dir) extra = manifest.get("file_pins", {}) sources: list[InputSource] = [] for raw in manifest["sources"]: @@ -230,13 +235,6 @@ def load_input_sources( "consumer_facts.jsonl": feed["facts_sha256"], }[member] files.append(PinnedFile(member, digest)) - elif key == "incumbent_efrs": - if member != reference["filename"]: - raise CutError( - f"incumbent member {member} is not the gate's " - f"reference {reference['filename']}." - ) - files.append(PinnedFile(member, reference["sha256"])) elif member in extra: files.append(PinnedFile(member, extra[member]["sha256"])) elif member in stage_pins: @@ -255,8 +253,6 @@ def load_input_sources( raise CutError( "frs_2024_25 revision disagrees with uk/frs_release.json." ) - if raw.get("revision_pin") == "gates:input_mass_parity": - revision = reference["revision"] if raw["kind"] == "url" and not url: raise CutError(f"{key}: the pin carries no https URL.") sources.append( @@ -604,6 +600,21 @@ def spine_h5(self) -> Path: def run_dir(self) -> Path: return self.cut / "national" + @property + def reference_dir(self) -> Path: + return self.cut / "reference" + + @property + def reference_record(self) -> Path: + return self.reference_dir / UK_CUT_REFERENCE_RECORD_NAME + + def reference_h5(self, reference: UKCutReference) -> Path: + return self.reference_dir / reference.filename + + @property + def score_receipt(self) -> Path: + return self.run_dir / "score_vs_reference.json" + @property def candidate_h5(self) -> Path: return self.run_dir / f"{CANDIDATE_NAME}.h5" @@ -675,11 +686,11 @@ def national_argv( python: str, spine_sha256: str, sources: Mapping[str, InputSource], + reference: UKCutReference, dry_run: bool, logbook_prev_row_digest: str | None = None, ) -> list[str]: chronicle = {pin.name: pin.sha256 for pin in sources["chronicle_uk"].files} - incumbent = sources["incumbent_efrs"].files[0] argv = [ python, str(REPO_ROOT / "tools/build_uk_full.py"), @@ -689,8 +700,8 @@ def national_argv( "--ledger-facts", str(paths.inputs / "chronicle_uk"), "--ledger-facts-sha256", chronicle["consumer_facts.jsonl"], "--ledger-manifest-sha256", chronicle["manifest.json"], - "--incumbent-h5", str(paths.input_file("incumbent", incumbent.name)), - "--incumbent-sha256", incumbent.sha256, + "--reference-h5", str(paths.reference_h5(reference)), + "--reference-record", str(paths.reference_record), # The build otherwise submits its graph to the public Orrery whenever # staging is remote; the graph files stay in the run directory. "--no-publish-orrery", @@ -704,20 +715,19 @@ def national_argv( def score_argv( - paths: CutPaths, *, python: str, candidate_sha256: str, sources: Mapping + paths: CutPaths, *, python: str, candidate_sha256: str, reference: UKCutReference ) -> list[str]: - incumbent = sources["incumbent_efrs"].files[0] return [ python, str(REPO_ROOT / "tools/score_uk_national_candidate.py"), "--candidate-h5", str(paths.candidate_h5), "--candidate-sha256", candidate_sha256, - "--incumbent-h5", str(paths.input_file("incumbent", incumbent.name)), - "--incumbent-sha256", incumbent.sha256, + "--reference-h5", str(paths.reference_h5(reference)), + "--reference-record", str(paths.reference_record), "--registry-json", str(paths.run_dir / "national_target_registry.json"), "--band-edge-registry-json", str(paths.run_dir / "national_contract_registry.json"), - "--output-json", str(paths.run_dir / "score_vs_incumbent.json"), + "--output-json", str(paths.score_receipt), ] # fmt: skip @@ -749,7 +759,8 @@ def certify_argv( "--ledger-manifest-sha256", chronicle["manifest.json"], "--input-mass-reference", str(paths.input_file("input_mass_reference", reference.name)), - "--score-receipt", str(paths.run_dir / "score_vs_incumbent.json"), + "--score-receipt", str(paths.score_receipt), + "--reference-record", str(paths.reference_record), "--release-id", RELEASE_ID, ] # fmt: skip if logbook_prev_row_digest: @@ -771,7 +782,7 @@ def assemble_argv(paths: CutPaths, *, python: str) -> list[str]: str(paths.run_dir / f"{CANDIDATE_NAME}.terminal_gates.json"), "--release-cut-gate-json", str(paths.run_dir / f"{CANDIDATE_NAME}.release_cut_gates.json"), - "--score-receipt", str(paths.run_dir / "score_vs_incumbent.json"), + "--score-receipt", str(paths.score_receipt), "--out-dir", "releases", ] # fmt: skip @@ -869,19 +880,51 @@ def candidate_sha256(paths: CutPaths) -> str: return str(record["artifacts"]["staging_h5"]["sha256"]) +def reference_done(paths: CutPaths) -> tuple[bool, str]: + """The cut's reference: a record, and the file it names.""" + + if not paths.reference_record.exists(): + return False, "no reference yet" + try: + reference = read_uk_cut_reference(paths.reference_record) + verify_uk_cut_reference_file(reference, paths.reference_h5(reference)) + except ValueError as error: + return False, str(error) + return True, f"compared with {reference.label}" + + +def cut_reference(paths: CutPaths) -> UKCutReference: + """The reference the cut recorded; a step that needs it refuses without it.""" + + done, detail = reference_done(paths) + if not done: + raise CutError(f"the cut has no usable reference: {detail}.") + return read_uk_cut_reference(paths.reference_record) + + +def _names_the_reference(record: object, reference: UKCutReference) -> bool: + return isinstance(record, Mapping) and ( + record.get("release"), + record.get("build"), + record.get("sha256"), + ) == (reference.release, reference.build, reference.sha256) + + def score_done(paths: CutPaths) -> tuple[bool, str]: - """The fields the certifier reads: the scored candidate and the verdict.""" + """The fields the certifier reads: the candidate and the reference scored.""" - receipt = _read_json(paths.run_dir / "score_vs_incumbent.json") + receipt = _read_json(paths.score_receipt) if receipt is None: return False, "no score receipt" scored = ((receipt.get("artifacts") or {}).get("candidate") or {}).get("sha256") if scored != candidate_sha256(paths): return False, "score receipt names another candidate" - verdict = (receipt.get("evaluation") or {}).get("verdict") - if verdict != "passed": - return False, f"score verdict is {verdict!r}" - return True, "score passed" + if (receipt.get("evaluation") or {}).get("schema_version") != 2: + return False, "score receipt is not a head-to-head with the reference" + reference = cut_reference(paths) + if not _names_the_reference(receipt.get("reference"), reference): + return False, "score receipt compared with another reference" + return True, f"compared with {reference.label}" def certify_done(paths: CutPaths) -> tuple[bool, str]: @@ -892,6 +935,8 @@ def certify_done(paths: CutPaths) -> tuple[bool, str]: return False, "certification is not shippable" if (cert.get("candidate") or {}).get("sha256") != candidate_sha256(paths): return False, "certification names another candidate" + if not _names_the_reference(cert.get("reference"), cut_reference(paths)): + return False, "certification names another reference" return True, "certified shippable" @@ -1467,6 +1512,47 @@ def run_spine(runner: Runner, *, staging_local_only: bool, retry: bool = False) raise CutError(f"the spine finished but is not usable: {detail}") +def run_reference( + runner: Runner, *, re_resolve: bool = False, hub_download=None +) -> UKCutReference: + """Resolve the current national build once per cut, and keep its record.""" + + paths = runner.paths + if re_resolve and paths.reference_dir.exists(): + aside = paths.reference_dir.with_name( + f"reference.superseded-{datetime.now(UTC):%Y%m%dT%H%M%S%fZ}" + ) + paths.reference_dir.rename(aside) + runner.log(f"[reference] kept the earlier reference as {aside.name}") + done, detail = reference_done(paths) + if done: + runner.log(f"[reference] done: {detail}") + return read_uk_cut_reference(paths.reference_record) + if paths.reference_dir.exists(): + raise CutError( + f"{paths.reference_dir} holds an unusable reference ({detail}); pass " + "--re-resolve-reference to set it aside and resolve the current " + "build again." + ) + runner.record("reference", status="running", started_at=_now()) + try: + reference = fetch_uk_cut_reference( + RELEASE_ID, paths.reference_dir, hub_download=hub_download + ) + except ValueError as error: + runner.record("reference", status="failed", error=str(error)[:600]) + raise CutError(f"the cut has no reference to compare with: {error}") from error + runner.record( + "reference", + status="done", + finished_at=_now(), + reference=reference.label, + sha256=reference.sha256, + ) + runner.log(f"[reference] compared with {reference.label} ({reference.sha256})") + return reference + + def run_national( runner: Runner, *, retry: bool, logbook_prev_row_digest: str | None ) -> None: @@ -1493,6 +1579,7 @@ def run_national( python=runner.python, spine_sha256=spine, sources=runner.sources, + reference=cut_reference(paths), logbook_prev_row_digest=logbook_prev_row_digest, ) dry_run = runner.state()["steps"].get("national-dry-run", {}) @@ -1511,19 +1598,19 @@ def run_score(runner: Runner) -> None: runner.log(f"[score] done: {detail}") return runner.log(f"[score] re-scoring the candidate ({detail})") - _set_aside(runner, "score", paths.run_dir / "score_vs_incumbent.json") + _set_aside(runner, "score", paths.score_receipt) runner.run( "score", score_argv( paths, python=runner.python, candidate_sha256=candidate_sha256(paths), - sources=runner.sources, + reference=cut_reference(paths), ), ) done, detail = score_done(paths) if not done: - raise CutError(f"the candidate does not pass the incumbent score: {detail}") + raise CutError(f"the comparison with the reference is not usable: {detail}") def national_logbook_row(paths: CutPaths) -> str: @@ -1591,6 +1678,10 @@ def stage1_done(paths: CutPaths, previous_h5: Path) -> tuple[bool, str]: def run_stage1(runner: Runner, previous_h5: Path | None) -> None: paths = runner.paths output = paths.diagnostics / "stage1_comparison.json" + if previous_h5 is None and reference_done(paths)[0]: + # The interim stage-1 comparison (microcosm#1276) is with the cut's + # own reference unless the operator names another file. + previous_h5 = paths.reference_h5(cut_reference(paths)) if previous_h5 is not None: done, detail = stage1_done(paths, previous_h5) if done: @@ -1600,8 +1691,8 @@ def run_stage1(runner: Runner, previous_h5: Path | None) -> None: if previous_h5 is None: runner.record("stage1", status="skipped", reason="no --previous-h5") runner.log( - "[stage1] skipped: pass --previous-h5 for the " - "interim stage-1 comparison (diagnostic; microcosm#1276)" + "[stage1] skipped: the cut has no reference; pass --previous-h5 for " + "the interim stage-1 comparison (diagnostic; microcosm#1276)" ) return paths.diagnostics.mkdir(parents=True, exist_ok=True) @@ -1712,6 +1803,11 @@ def write_receipt(runner: Runner, extra: Mapping[str, Any]) -> None: } for source in runner.sources.values() }, + "reference": ( + read_uk_cut_reference(paths.reference_record).to_record() + if paths.reference_record.exists() + else None + ), "spine_sha256": spine_sha256(paths), "candidate_sha256": candidate_sha256(paths) if (paths.run_dir / "build_record.json").exists() @@ -1779,7 +1875,19 @@ def _parse(argv: Sequence[str] | None) -> argparse.Namespace: help="move an unfinished spine attempt aside and build a fresh one", ) parser.add_argument("--retry-national", action="store_true") - parser.add_argument("--previous-h5", type=Path, help="for the stage-1 comparison") + parser.add_argument( + "--previous-h5", + type=Path, + help="for the stage-1 comparison; by default the cut's reference", + ) + parser.add_argument( + "--re-resolve-reference", + action="store_true", + help=( + "set the cut's reference aside and compare with the national " + "line's current build again" + ), + ) parser.add_argument( "--logbook-prev-row-digest", help=( @@ -1860,9 +1968,10 @@ def preflight(args: argparse.Namespace, paths: CutPaths) -> list[Check]: return checks -def status(paths: CutPaths) -> None: +def status(paths: CutPaths, *, check_current=None) -> None: print(f"cut {paths.cut}") for name, check in ( + ("reference", reference_done), ("spine", spine_done), ("national", national_done), ("score", score_done), @@ -1874,6 +1983,24 @@ def status(paths: CutPaths) -> None: except Exception as error: # an earlier step is missing done, detail = False, f"not reached ({type(error).__name__})" print(f" {'done' if done else 'todo'} {name}: {detail}") + if paths.reference_record.exists(): + print(f" {_current_build_note(paths, check_current)}") + + +def _current_build_note(paths: CutPaths, check_current=None) -> str: + """Whether the national line's current build has moved since the fetch.""" + + recorded = read_uk_cut_reference(paths.reference_record) + try: + current = (check_current or check_uk_cut_reference)(RELEASE_ID) + except Exception as error: # the Hub's answer is only a note here + return f"note: could not read the current build ({type(error).__name__})" + if current.get("build") == recorded.build: + return "note: the reference is still the current national build" + return ( + f"note: the current national build is now {current.get('build')}; this " + f"cut compares with {recorded.build} (--re-resolve-reference to move it)" + ) def main(argv: Sequence[str] | None = None) -> int: @@ -1922,6 +2049,7 @@ def main(argv: Sequence[str] | None = None) -> int: if args.command == "preflight": return 0 runner = Runner(paths=paths, sources=sources) + run_reference(runner, re_resolve=args.re_resolve_reference) run_spine( runner, staging_local_only=not args.spine_telemetry, retry=args.retry_spine ) diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index 7657454d9d..589b2440bb 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -1,10 +1,18 @@ -"""Score a UK national candidate against the incumbent on one frozen register. +"""Compare a UK national candidate with the reference on one frozen register. -The score core lives in ``microcosm.build.uk_runtime.candidate_score``; this -command loads the frozen register, verifies both artifacts against their -digests, scores on the surface both can materialize (pruning the rows the -incumbent cannot, loudly, unless ``--no-prune-incumbent-unresolvable``) and +The comparison core lives in ``microcosm.build.uk_runtime.candidate_score`` +(over the country-neutral ``microcosm.build.head_to_head``); this command +loads the frozen register, verifies both artifacts against their digests, +compares them on the surface both can materialize (pruning the rows the +reference cannot, loudly, unless ``--no-prune-reference-unresolvable``) and writes the receipt the release-cut certifier reads. + +The reference is the current national build a cut recorded when it fetched +it (microcosm#1095): pass ``--reference-record`` with its file. An ad hoc +comparison may name a file with ``--reference-sha256`` and +``--reference-label`` instead; its receipt carries no reference record, so +it never certifies a cut. The comparison decides nothing: it has no +threshold, like the US head-to-head. """ from __future__ import annotations @@ -24,11 +32,8 @@ _relative_errors, _scored_frame, _sha256_file, - _target_drift, - _target_wins, - _target_wins_by_family, _verify_artifact, - evaluate_uk_candidate_against_incumbent, + evaluate_uk_candidate_against_reference, pruned_warning, score_uk_national_candidate, ) @@ -40,12 +45,47 @@ def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: return output +def _reference(args: argparse.Namespace, parser: argparse.ArgumentParser): + """The reference's sha, label and record, from the record when given.""" + + if args.reference_record is not None: + if args.reference_sha256 is not None or args.reference_label is not None: + parser.error( + "--reference-record names the reference; drop --reference-sha256 " + "and --reference-label." + ) + from microcosm.build.uk_runtime.cut_reference import ( + read_uk_cut_reference, + verify_uk_cut_reference_file, + ) + + record = read_uk_cut_reference(args.reference_record) + verify_uk_cut_reference_file(record, args.reference_h5) + return record.sha256, record.label, record.to_record() + if args.reference_sha256 is None or args.reference_label is None: + parser.error( + "pass --reference-record, or both --reference-sha256 and " + "--reference-label for an ad hoc comparison." + ) + return args.reference_sha256, args.reference_label, None + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-h5", required=True, type=Path) parser.add_argument("--candidate-sha256", required=True) - parser.add_argument("--incumbent-h5", required=True, type=Path) - parser.add_argument("--incumbent-sha256", required=True) + parser.add_argument("--reference-h5", required=True, type=Path) + parser.add_argument( + "--reference-record", + type=Path, + help="The cut's record of the reference build (tools/resolve_uk_reference.py).", + ) + parser.add_argument( + "--reference-sha256", help="Ad hoc only: the reference file's SHA-256." + ) + parser.add_argument( + "--reference-label", help="Ad hoc only: the reference's label in the receipt." + ) parser.add_argument("--registry-json", required=True, type=Path) parser.add_argument("--output-json", required=True, type=Path) parser.add_argument("--calibration-year", type=int) @@ -53,11 +93,6 @@ def main(argv: list[str] | None = None) -> int: "--candidate-label", help="Override the candidate label; by default it is the candidate H5 stem.", ) - parser.add_argument( - "--incumbent-label", - default="enhanced_frs_2024_25", - help="Override the incumbent/reference label.", - ) parser.add_argument( "--band-edge-registry-json", type=Path, @@ -68,12 +103,13 @@ def main(argv: list[str] | None = None) -> int: ), ) parser.add_argument( - "--no-prune-incumbent-unresolvable", + "--no-prune-reference-unresolvable", action="store_true", help=( - "Refuse the score when the incumbent cannot materialize a target " - "instead of pruning that target from both arms and reporting it. " - "The default prunes and warns, naming every absent measure." + "Refuse the comparison when the reference cannot materialize a " + "target instead of pruning that target from both arms and " + "reporting it. The default prunes and warns, naming every absent " + "measure." ), ) parser.add_argument( @@ -86,6 +122,7 @@ def main(argv: list[str] | None = None) -> int: ), ) args = parser.parse_args(argv) + reference_sha256, reference_label, reference_record = _reference(args, parser) registry = _load_registry(args.registry_json) band_edge_registry = ( None @@ -104,34 +141,38 @@ def main(argv: list[str] | None = None) -> int: args.output_json.parent, int(calibration_year) ) ) - score = evaluate_uk_candidate_against_incumbent( + score = evaluate_uk_candidate_against_reference( candidate_h5=args.candidate_h5, - incumbent_h5=args.incumbent_h5, + reference_h5=args.reference_h5, candidate_sha256=args.candidate_sha256, - incumbent_sha256=args.incumbent_sha256, + reference_sha256=reference_sha256, target_registry=registry, calibration_year=int(calibration_year), + reference_label=reference_label, measure_resolver_factory=factory, candidate_label=args.candidate_label, - incumbent_label=args.incumbent_label, band_edge_registry=band_edge_registry, - prune_incumbent_unresolvable_measures=not args.no_prune_incumbent_unresolvable, + prune_reference_unresolvable_measures=not args.no_prune_reference_unresolvable, + reference_record=reference_record, ) warning = pruned_warning(score) if warning is not None: print(warning, file=sys.stderr, flush=True) # The receipt is the score block itself: the release-cut certifier reads - # ``artifacts.candidate.sha256`` and ``evaluation.verdict`` at the top - # level, and the assembler copies the bytes into the release directory - # unchanged. + # ``artifacts.candidate.sha256`` and ``reference`` at the top level, and + # the assembler copies the bytes into the release directory unchanged. _write_json(args.output_json, score) evaluation = score["evaluation"] + targets = evaluation["targets"] print( - f"evaluation {evaluation['verdict']}: candidate loss " - f"{evaluation['rule_1']['candidate_full_loss']:.6f} vs incumbent " - f"{evaluation['rule_1']['incumbent_full_loss']:.6f} on " + f"comparison with {evaluation['reference']} (no threshold): candidate " + f"loss {evaluation['losses']['candidate_full_loss']:.6f} vs reference " + f"{evaluation['losses']['reference_full_loss']:.6f} on " f"{evaluation['scored_surface']['n_scored']} of " - f"{evaluation['scored_surface']['n_surface']} targets", + f"{evaluation['scored_surface']['n_surface']} targets; lower error on " + f"{targets['candidate_lower_count']} for the candidate, " + f"{targets['reference_lower_count']} for the reference, " + f"{targets['equal_count']} equal", flush=True, ) return 0 diff --git a/tools/uk_cut_inputs.json b/tools/uk_cut_inputs.json index b5eccd5e88..83ffcbcb47 100644 --- a/tools/uk_cut_inputs.json +++ b/tools/uk_cut_inputs.json @@ -1,6 +1,6 @@ { "version": 1, - "about": "Where tools/run_uk_cut.py fetches each input of a Microcosm UK national cut. This file says only where; every expected sha256 and size is read from the committed pins (uk/source_stages.json, uk/frs_release.json, uk/chronicle_feed.json, uk/gates.json, weighted_integrity.py), so a file is accepted only if it is byte-identical to what the build itself will check. A *_pin field reads its value from those pins instead: path_in_repo_pin chronicle_feed puts the Chronicle artifact under chronicle-uk/, so it follows every re-pin. revision null reads the repository's main branch, and every file is still verified against its pin (docs/uk-cut-anyone.md).", + "about": "Where tools/run_uk_cut.py fetches each input of a Microcosm UK national cut. This file says only where; every expected sha256 and size is read from the committed pins (uk/source_stages.json, uk/frs_release.json, uk/chronicle_feed.json, weighted_integrity.py), so a file is accepted only if it is byte-identical to what the build itself will check. A *_pin field reads its value from those pins instead: path_in_repo_pin chronicle_feed puts the Chronicle artifact under chronicle-uk/, so it follows every re-pin. revision null reads the repository's main branch, and every file is still verified against its pin (docs/uk-cut-anyone.md).", "sources": [ { "key": "frs_2024_25", @@ -150,18 +150,6 @@ "members": [ "uk_input_mass_reference.json" ] - }, - { - "key": "incumbent_efrs", - "kind": "hf_files", - "repo_id": "policyengine/policyengine-uk-data-private", - "repo_type": "model", - "revision_pin": "gates:input_mass_parity", - "path_in_repo": "", - "directory": "incumbent", - "members": [ - "enhanced_frs_2024_25.h5" - ] } ], "file_pins": { From 1da62c183903528400169dd70ea822f37fb6ae88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:42:46 +0000 Subject: [PATCH 04/18] Fence UK input mass against the current build of the cut's line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The input-mass gate compared every cut with the enhanced FRS's weighted totals (microcosm#1095). It now compares with the current build of the cut's own line, the build the cut's reference step resolved. - uk/gates.json names the reference kind, current_build, not a file. The report records the build (reference_label, reference_identity) and its totals digest. The tolerance is 5.9984285545209834: the exact maximum drift between the consecutive promoted national cuts a6299504 and eeb6093e over the 259 columns at least three source households carry, with no headroom, as María signed on 2026-11-10. A zeroed or missing column still fails at any tolerance. - The reference step (run_uk_cut, and tools/resolve_uk_reference.py for a dense build) writes input_mass_reference.json beside the reference: schema 2, the build's export totals and distinct source-household carriers at its own solved weights. uk_recorded_solve_year_frame gives a build back its solve by the factor its mass log records, since an older policyengine-uk's projection differs from the installed one. - A column one or two source households carry on either side is still checked, but its figures leave the report (withheld_thin_columns). - Exclusions are signed per reference build (@) and go dormant when a promotion moves past it. The enhanced FRS's three entries, its registry entry, the hosted totals input and their pins retire; microcosm-data's contract checks that a report names its build instead of pinning the eFRS identity and digests. Digests re-pinned. - certify refuses an input-mass reference that is not the cut's reference. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/uk/gates.json | 18 +- .../uk/input_mass_reviewed_exclusions.json | 30 +- .../build/uk_runtime/battery_bindings.py | 39 +- .../build/uk_runtime/cut_reference.py | 58 +++ .../microcosm/build/uk_runtime/weight_year.py | 37 ++ .../build/uk_runtime/weighted_integrity.py | 404 ++++++++++++++---- .../engine/uk/test_uk_efrs_weighted_totals.py | 58 +-- .../engine_free/shared/test_country_spec.py | 5 +- .../shared/test_gate_battery_contract_pins.py | 35 +- .../uk/test_resolve_uk_reference.py | 11 +- .../tests/engine_free/uk/test_run_uk_cut.py | 78 +++- .../uk/test_uk_battery_bindings.py | 106 +++-- .../uk/test_uk_dense_signoff_draft.py | 6 +- .../engine_free/uk/test_uk_terminal_gates.py | 69 ++- .../uk/test_uk_weighted_integrity.py | 277 +++++------- .../src/microcosm/data/contract.py | 103 +++-- .../tests/engine_free/shared/test_contract.py | 109 ++--- .../microcosm_build/uk_cut_reference.py | 33 ++ tools/build_uk_efrs_parity_reference.py | 5 +- tools/certify_uk_release_cut.py | 21 +- tools/resolve_uk_reference.py | 38 +- tools/run_uk_cut.py | 40 +- tools/uk_cut_inputs.json | 23 +- 23 files changed, 990 insertions(+), 613 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index fb6f9c2436..8d1f4d4240 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -1824,24 +1824,18 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": { - "reference": "efrs-post-calibration", + "reference": "current_build", "reference_registry": { - "efrs-post-calibration": { - "identity": { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", - "vintage": "2024_25" - }, - "totals_sha256": "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", - "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; its artifact carries the SPI-synthetic rows structurally but no admin-restored mass in the SPI-channel-exclusive columns, so those columns are comparable only through per-reference reviewed exclusions." + "current_build": { + "kind": "current_build", + "scope_note": "The current build of the cut's own release line, as releases.json named it when the cut fetched its inputs (microcosm#1095): totals at each build's own solved weights, on the export surface." } }, - "relative_tolerance": 4.521811483823806, + "relative_tolerance": 5.9984285545209834, "minimum_reference_total": 0.0, "reviewed_exclusions_resource": "input_mass_reviewed_exclusions.json" }, - "notes": "Weighted input-mass parity against the active named reference in the registry, with reviewed exclusions scoped per reference so a channel-blind reference exclusion does not suppress comparison against a future channel-aware reference. The tolerance is the worst surviving measured drift at the reviewed floor, exact, with no headroom; the reviewed per-reference exclusions (charitable_investment_gifts, signed at +15,168.8% on the post-#710 surface; owned_land, signed for seed variance) were retired on 2026-09-22 when the first release-cut certification measured both inside the fence (-99.0% and -73.5% against the enhanced FRS reference, microcosm#967); the register is empty and the gate's current failures are the two genuine breaches it names. The 0.0 floor excludes only arithmetically meaningless zero-mass denominators. Totals stay uncommitted under the UKDS licence, supplied per-run and pinned by totals_sha256; missing sidecar evidence records evidence_absent. Totals keys are flat frame column names since the #611 A4 dedup; that digest re-freeze was keys-only against the #610 values (identity unchanged, values byte-identical). Sensitivity floor, recorded rather than silent (microcosm#706 review): the tolerance is the worst surviving measured drift, and that edge is currently set by a column whose deviation is explained (vintage plus SPI composition, microcosm#703's head-to-head receipt), so at this value the gate is a gross mass-loss/explosion fence rather than a regression detector — the compared population sits at median 0.24 and p90 0.63. Tightening runs through this register: each explained top drift that earns a receipt lets the tolerance re-mint at the worst unexplained survivor, with the full distribution and the exclusions-versus-tolerance tradeoff in experiments/630-uk-gate-adjudication-receipts.md." + "notes": "Weighted input-mass parity against the current build of the cut's own line (microcosm#1095, replacing the enhanced-FRS reference): the national line against the national current build, the dense line against the dense one. The spec names the reference kind, not a file; each report records the build it compared with, its identity and the totals digest. A zeroed or missing column fails at any tolerance, the #278 signature. The tolerance is the exact maximum drift between the consecutive promoted national cuts a6299504 and eeb6093e over the 259 columns carried by at least three distinct source households, with no headroom (median 0.020, p90 0.22, p99 1.06; none zeroed or dropped), signed by María on 2026-11-10; it ratchets down as each newly certified pair is measured, the release-arm method of weighted_integrity.py. Reviewed exclusions are signed for one reference build and go dormant once a promotion moves the reference past it. A column one or two source households carry on either side is checked but its figures are withheld from the report. The totals are licensed-data aggregates written beside the cut's reference, never committed; missing evidence records evidence_absent. The 0.0 floor excludes only arithmetically meaningless zero-mass denominators." }, { "id": "uk_local_geography_ladder_post_calibration", diff --git a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json index b1511d68ed..0fa63858f6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json @@ -1,29 +1,5 @@ { - "schema_version": 3, - "description": "Reviewed input-mass parity exclusions for the UK terminal battery, scoped per named comparison reference (microcosm#630): an exclusion suppresses only when its reference is the armed one, so a column excluded on a channel-blind reference is compared normally against a channel-aware one with no register change. Each entry is a complete approval receipt (schema-2 fields per #610). The gate reports dormant entries, FAILS stale ones, and FAILS expired ones until the adjudication is renewed, so this register cannot rot.", - "references": { - "efrs-post-calibration": { - "dfe_education_spending": { - "reason": "The candidate carries DfE education spending through the build (GBP 98.8bn against the incumbent's GBP 52.8m, +187,061%): the enhanced-FRS reference carries almost none of this input, so the comparison measures the reference's omission, not the candidate's. The input stays on the release surface and in the coverage manifest.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1063 c9 (certifier rehearsal R5 on the 2026-09-30 build, 2026-10-01; ruling 2026-10-02 in session)", - "approved_on": "2026-10-02", - "expires_on": "2027-04-02" - }, - "property_wealth": { - "reason": "The candidate no longer ships household.property_wealth: policyengine-uk derives it as the sum of main_residence_value, other_residential_property_value and non_residential_property_value, and a persisted copy overrides that sum and has no uprating index (microcosm#1106, the uk-data#543 defect). The enhanced-FRS reference stores its own WAS total, so the column is present in the reference and absent from the candidate, which this gate otherwise fails whatever the tolerance. The gate compares the release surface (the export frame, reviewed exclusions dropped) on both paths, the national seam and the release cut; the three components stay compared.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1106 (#1145): part B measured on the A9 candidate at 2025 and on the final-head arm c7 of 2026-10-11; ruling 2026-10-11 in session", - "approved_on": "2026-10-11", - "expires_on": "2027-04-11" - }, - "tax_free_savings_income": { - "reason": "The candidate no longer ships person.tax_free_savings_income: policyengine-uk derives it as the sum of individual_savings_account_interest_income, which the candidate stores apart from non-ISA interest under the additive ISA income contract, and a persisted copy overrides that sum and is never uprated (microcosm#1360). The enhanced-FRS reference stores it in the legacy gross layout, so the column is present in the reference and absent from the candidate, which this gate otherwise fails whatever the tolerance. Non-ISA savings_interest_income stays compared; it falls below the reference's gross total by the ISA part.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1360 folded into microcosm#1418's PR B (ruling 2026-10-28 in session)", - "approved_on": "2026-10-28", - "expires_on": "2027-04-28" - } - } - } + "schema_version": 3, + "description": "Reviewed input-mass parity exclusions for the UK battery, scoped per reference build (microcosm#1095): each key is the @ a cut compared with, and an exclusion suppresses only while that build is the cut's reference, so a promotion leaves it dormant. Each entry is a complete approval receipt (schema-2 fields per #610). The gate reports dormant entries, FAILS stale ones, and FAILS expired ones until the adjudication is renewed. The enhanced-FRS reference's three entries retired with it.", + "references": {} } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index d8a28f2910..ab7cba76f3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -124,6 +124,7 @@ coerce_reviewed_exclusions, load_uk_local_area_support_exclusion_register, uk_default_input_mass_reviewed_exclusions, + uk_input_mass_carriers, uk_input_mass_parity_gate, uk_input_mass_totals, ) @@ -1143,18 +1144,31 @@ def _validate_input_mass_exclusion_references( *, label: str, ) -> None: - unknown = sorted(set(register) - set(UK_INPUT_MASS_REFERENCE_REGISTRY)) - if unknown: + from microcosm.build.uk_runtime.cut_reference import UK_CUT_REFERENCE_RELEASES + + malformed = sorted( + name + for name in register + if name.count("@") != 1 + or name.split("@", 1)[0] not in UK_CUT_REFERENCE_RELEASES + or not name.split("@", 1)[1] + ) + if malformed: raise ValueError( - f"{label} names unknown input-mass reference(s) {unknown}; known " - f"references are {sorted(UK_INPUT_MASS_REFERENCE_REGISTRY)}." + f"{label} names {malformed}; an input-mass exclusion is signed for " + "one reference build, named @ for a UK line " + "(microcosm#1095)." ) def _resolve_input_mass_exclusions( context: EvidenceContext, *, reference: str ) -> tuple[Mapping[str, UKReviewedExclusion], str]: - """Active-reference input-mass exclusions and their content source.""" + """The exclusions signed for this reference build, and their source. + + ``reference`` is the build's ``@``; entries signed for + any other build are dormant (microcosm#1095). + """ committed = uk_default_input_mass_reviewed_exclusions() _validate_input_mass_exclusion_references( @@ -1780,13 +1794,15 @@ def _evaluate_input_mass_parity( f"uk/gates.json declares input-mass reference {reference_name!r} " f"but known references are {sorted(coerced_registry)}." ) + reference = context.artifacts["input_mass_reference"] + descriptor = coerced_registry[reference_name].descriptor_for(reference) if register != UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE: raise ValueError( f"uk/gates.json names exclusion register {register!r} but the " f"runtime loads {UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE!r}." ) exclusions, _source = _resolve_input_mass_exclusions( - context, reference=reference_name + context, reference=reference.label ) exclusions, out_of_line = _in_line(exclusions, context) policy = UKInputMassParityPolicy( @@ -1800,12 +1816,14 @@ def _evaluate_input_mass_parity( # carries the spine's WAS property_wealth, which the reference also # stores, so comparing it there would call the signed exclusion stale # (microcosm#1106). + export_frame = uk_release_export_frame(context.frame) result = uk_input_mass_parity_gate( - uk_input_mass_totals(uk_release_export_frame(context.frame)), - context.artifacts["input_mass_reference"], - descriptor=coerced_registry[reference_name], + uk_input_mass_totals(export_frame), + reference, + descriptor=descriptor, policy=policy, now=_exclusion_clock(context), + candidate_carriers=uk_input_mass_carriers(export_frame), **kwargs, ) return _with_out_of_line(result, context, out_of_line) @@ -1817,10 +1835,11 @@ def _input_mass_reference_evidence( reference_name = str(parameters["reference"]) reference = context.artifacts["input_mass_reference"] exclusions, source = _resolve_input_mass_exclusions( - context, reference=reference_name + context, reference=reference.label ) return { "reference": reference_name, + "reference_label": reference.label, "reference_evidence_sha256": _input_mass_reference_evidence_sha256(reference), "exclusions_policy": source, "reviewed_exclusions": _exclusion_payload(exclusions), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py index bb41a27794..d60438e064 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py @@ -41,19 +41,25 @@ "UK_CUT_REFERENCE_RELEASES", "UK_CUT_REFERENCE_SCHEMA_VERSION", "UKCutReference", + "UK_CUT_REFERENCE_INPUT_MASS_NAME", "UK_CUT_REFERENCE_RECORD_NAME", "check_uk_cut_reference", "fetch_uk_cut_reference", "read_uk_cut_reference", "resolve_uk_cut_reference", + "uk_cut_reference_evidence_done", "verify_uk_cut_reference_file", "write_uk_cut_reference", + "write_uk_cut_reference_evidence", ] UK_CUT_REFERENCE_KIND = "uk_cut_reference" UK_CUT_REFERENCE_SCHEMA_VERSION = 1 #: The record's file name beside the reference's dataset in a cut. UK_CUT_REFERENCE_RECORD_NAME = "reference.json" +#: The reference's input-mass totals and carriers, written beside it: the +#: input-mass gate's reference (licensed-data aggregates, never committed). +UK_CUT_REFERENCE_INPUT_MASS_NAME = "input_mass_reference.json" #: Each line's registry variant and the build evidence its fetch reads. UK_CUT_REFERENCE_RELEASES: Mapping[str, tuple[str, str]] = { @@ -463,3 +469,55 @@ def fetch_uk_cut_reference( partial.replace(destination) write_uk_cut_reference(record_path, reference) return reference + + +def write_uk_cut_reference_evidence( + reference: UKCutReference, h5_path: str | Path, output_dir: str | Path +) -> dict[str, Path]: + """Write the evidence the gates read from the reference, beside it. + + The input-mass totals and carriers of the build's export frame at its own + solved weights (:func:`~.weighted_integrity.uk_input_mass_reference_payload`). + The file holds licensed-data aggregates: it stays with the cut, as the + reference's dataset does. + """ + + from microcosm.build.uk_runtime.weighted_integrity import ( + uk_input_mass_reference_payload, + ) + + source = verify_uk_cut_reference_file(reference, h5_path) + directory = Path(output_dir) + directory.mkdir(parents=True, exist_ok=True) + payload = uk_input_mass_reference_payload( + source, + label=reference.label, + filename=reference.filename, + revision=reference.build, + sha256=reference.sha256, + ) + destination = directory / UK_CUT_REFERENCE_INPUT_MASS_NAME + partial = destination.with_name(destination.name + ".partial") + partial.write_text(json.dumps(payload, sort_keys=True, allow_nan=False)) + partial.replace(destination) + return {"input_mass_reference": destination} + + +def uk_cut_reference_evidence_done( + reference: UKCutReference, output_dir: str | Path +) -> tuple[bool, str]: + """Whether the reference's evidence is beside it and names this build.""" + + path = Path(output_dir) / UK_CUT_REFERENCE_INPUT_MASS_NAME + try: + payload = json.loads(path.read_text()) + except (OSError, ValueError): + return False, f"no {UK_CUT_REFERENCE_INPUT_MASS_NAME} beside the reference" + identity = payload.get("identity") if isinstance(payload, Mapping) else None + if ( + not isinstance(identity, Mapping) + or payload.get("label") != reference.label + or identity.get("sha256") != reference.sha256 + ): + return False, f"{UK_CUT_REFERENCE_INPUT_MASS_NAME} names another build" + return True, f"evidence for {reference.label}" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weight_year.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weight_year.py index 2a7e44a245..56bb5beaf9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weight_year.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weight_year.py @@ -57,6 +57,11 @@ "Household weights as policyengine-uk projects them from the dataset year " "to the simulation year (microcosm#796 A19)." ) +UK_RECORDED_SOLVE_YEAR_MASS_REASON = ( + "Household weights at the solve year by the release's own record: the " + "stored weights times the factor its build divided them by, whatever " + "policyengine-uk is installed (microcosm#1095)." +) def uk_weight_projection_growth(year: int) -> float: @@ -182,14 +187,46 @@ def uk_solve_year_frame(frame: Frame, *, solve_year: int) -> Frame: return uk_engine_year_frame(frame, simulation_year=solve_year) +def uk_recorded_solve_year_frame(frame: Frame) -> Frame: + """A written frame at the weights its own build solved, by its record. + + :func:`uk_solve_year_frame` gives a release back its solved weights + through the installed engine's projection, so it refuses a release an + earlier policyengine-uk wrote, whose projection differed. A cut's + reference is such a release: the current build may predate the installed + engine. Its mass log records the factor its build divided by, and that + factor alone returns its solve (microcosm#1095). A frame that was never + divided is returned as it is. + """ + + if not uk_weights_are_at_dataset_year(frame): + return frame + record = [r for r in frame.mass_log if r.entity == "household"][-1] + declared = record.declared_factor + if declared is None or not np.isfinite(float(declared)) or float(declared) <= 0.0: + raise ValueError( + "The frame's household weights were divided at the dataset year, but " + "its mass log records no positive factor to multiply them back by." + ) + weights = frame.weights_for("household") + factor = 1.0 / float(declared) + return frame.with_weights( + "household", + Weights(np.asarray(weights.values, dtype=np.float64) * factor, weights.kind), + mass=MassChange(factor=factor, reason=UK_RECORDED_SOLVE_YEAR_MASS_REASON), + ) + + __all__ = [ "UK_ENGINE_YEAR_MASS_REASON", + "UK_RECORDED_SOLVE_YEAR_MASS_REASON", "UK_WEIGHT_PROJECTION_PARAMETER", "UK_WEIGHT_PROJECTION_PINS_BLOCK", "UK_WEIGHT_PROJECTION_VARIABLE", "UK_WEIGHT_YEAR_MASS_REASON", "uk_dataset_year_frame", "uk_engine_year_frame", + "uk_recorded_solve_year_frame", "uk_solve_year_frame", "uk_weight_projection_factor", "uk_weight_projection_growth", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index fe7c1f697b..cd2566bbbf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -130,19 +130,20 @@ GateResult, input_mass_parity_gate, ) -from microcosm.build.input_mass import input_mass_totals +from microcosm.build.input_mass import PERSON_REFERENCE_COLUMNS, input_mass_totals from microcosm.build.uk_runtime.national_frame import UK_EXPORTED_WEIGHT_COLUMNS from microcosm.frame import Frame __all__ = [ "UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE", "UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE", + "UK_CURRENT_BUILD_REFERENCE", "UK_INPUT_MASS_PARITY_GATE_NAME", - "UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256", "UK_LOCAL_TARGET_FIT_EXCLUSION_REGISTER_RESOURCE", "UK_TARGET_FIT_EXCLUSION_REGISTER_RESOURCE", "UKInputMassParityPolicy", "UKInputMassReference", + "UKCurrentBuildReference", "UKInputMassReferenceDescriptor", "UKReviewedExclusion", "UK_INPUT_MASS_REFERENCE_REGISTRY", @@ -155,7 +156,9 @@ "load_uk_reference_scoped_exclusion_register", "load_uk_reviewed_exclusion_register", "uk_default_input_mass_reviewed_exclusions", + "uk_input_mass_carriers", "uk_input_mass_parity_gate", + "uk_input_mass_reference_payload", "uk_input_mass_totals", "uk_qrf_tail_concentration_columns", ] @@ -170,14 +173,11 @@ "local_target_fit_reviewed_exclusions.json" ) -# Canonical sha256 of {"reference": {"identity": ..., "totals": ...}} for -# the weighted input surface emitted from the pinned enhanced-FRS artifact by -# build_uk_efrs_parity_reference.py. The totals remain uncommitted under the -# UKDS EUL; this reviewed digest lets the gate and publication contract bind -# them without disclosing them. -UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" -) +#: The input-mass reference file a cut writes from its reference build +#: (microcosm#1095): schema 2 names the build and carries each column's +#: carriers. Schema 1 held the enhanced FRS's totals, which retired with it. +UK_INPUT_MASS_REFERENCE_SCHEMA_VERSION = 2 +UK_INPUT_MASS_REFERENCE_KIND = "uk_input_mass_reference" _SHA256_HEX = frozenset("0123456789abcdef") @@ -247,24 +247,69 @@ def spec_payload(self) -> dict[str, object]: } -_UK_INPUT_MASS_REFERENCE_DESCRIPTOR = UKInputMassReferenceDescriptor( - name="efrs-post-calibration", - filename="enhanced_frs_2024_25.h5", - revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - sha256="e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", - vintage="2024_25", - totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, +@dataclass(frozen=True) +class UKCurrentBuildReference: + """The input-mass reference kind: the current build of the cut's own line. + + The spec names the kind, not a file (microcosm#1095): a cut compares its + candidate with the build its line's ``releases.json`` named when the cut + fetched its inputs, and every report records which build that was. The + concrete descriptor is made from the reference file the cut wrote + (:meth:`descriptor_for`), so the identity and totals digest the gate + checks are the reference's own. + """ + + name: str + scope_note: str + + def __post_init__(self) -> None: + for field_name in ("name", "scope_note"): + value = getattr(self, field_name) + if ( + not isinstance(value, str) + or not value.strip() + or value != value.strip() + ): + raise ValueError( + f"UK input-mass reference {field_name} must be a non-empty " + "trimmed string." + ) + + def spec_payload(self) -> dict[str, object]: + """The gates.json registry entry shape.""" + + return {"kind": "current_build", "scope_note": self.scope_note} + + def descriptor_for( + self, reference: UKInputMassReference + ) -> UKInputMassReferenceDescriptor: + if reference.label is None: + raise ValueError( + "The input-mass reference names no build; a cut's reference file " + "records the build it was measured from." + ) + return UKInputMassReferenceDescriptor( + name=self.name, + filename=reference.filename, + revision=reference.revision, + sha256=reference.sha256, + vintage=reference.vintage, + totals_sha256=_input_mass_reference_evidence_sha256(reference), + scope_note=self.scope_note, + ) + + +UK_CURRENT_BUILD_REFERENCE = UKCurrentBuildReference( + name="current_build", scope_note=( - "Channel-blind post-calibration enhanced-FRS production incumbent, " - "pinned to the 2024-25 line; its artifact carries the SPI-synthetic " - "rows structurally but no admin-restored mass in the " - "SPI-channel-exclusive columns, so those columns are comparable " - "only through per-reference reviewed exclusions." + "The current build of the cut's own release line, as releases.json " + "named it when the cut fetched its inputs (microcosm#1095): totals " + "at each build's own solved weights, on the export surface." ), ) UK_INPUT_MASS_REFERENCE_REGISTRY = MappingProxyType( - {_UK_INPUT_MASS_REFERENCE_DESCRIPTOR.name: _UK_INPUT_MASS_REFERENCE_DESCRIPTOR} + {UK_CURRENT_BUILD_REFERENCE.name: UK_CURRENT_BUILD_REFERENCE} ) _REVIEWED_EXCLUSION_FIELDS = frozenset( @@ -466,57 +511,43 @@ def coerce_reviewed_exclusions( def coerce_input_mass_reference_registry( value: object, *, label: str -) -> dict[str, UKInputMassReferenceDescriptor]: - """Validate the closed-world input-mass reference registry.""" +) -> dict[str, UKCurrentBuildReference]: + """Validate the closed-world input-mass reference registry. + + Its one kind is the current build of the cut's own line + (``{"kind": "current_build", "scope_note": ...}``); a file pinned by + identity, as the enhanced-FRS entry was, is refused. + """ if not isinstance(value, Mapping) or not value: raise TypeError(f"{label} reference_registry must be a non-empty mapping.") - records: dict[str, UKInputMassReferenceDescriptor] = {} + records: dict[str, UKCurrentBuildReference] = {} for name, entry in value.items(): if not isinstance(name, str) or not name.strip() or name != name.strip(): raise ValueError( f"{label} reference_registry names must be non-empty trimmed " f"strings; got {name!r}." ) - if isinstance(entry, UKInputMassReferenceDescriptor): - descriptor = entry - if descriptor.name != name: + if isinstance(entry, UKCurrentBuildReference): + if entry.name != name: raise ValueError( f"{label} reference_registry entry {name!r} carries " - f"descriptor name {descriptor.name!r}." + f"reference name {entry.name!r}." ) - records[name] = descriptor + records[name] = entry continue if not isinstance(entry, Mapping): raise TypeError( f"{label} reference_registry entry {name!r} must be an object." ) - expected_entry_fields = {"identity", "totals_sha256", "scope_note"} - if set(entry) != expected_entry_fields: + if set(entry) != {"kind", "scope_note"} or entry.get("kind") != "current_build": raise ValueError( - f"{label} reference_registry entry {name!r} fields must be " - f"exactly {sorted(expected_entry_fields)}, got {sorted(entry)}." + f"{label} reference_registry entry {name!r} must be exactly " + '{"kind": "current_build", "scope_note": ...}; a reference pinned ' + "by file identity is no longer accepted (microcosm#1095)." ) - identity = entry["identity"] - if not isinstance(identity, Mapping): - raise TypeError( - f"{label} reference_registry entry {name!r} identity must be an object." - ) - expected_identity_fields = {"filename", "revision", "sha256", "vintage"} - if set(identity) != expected_identity_fields: - raise ValueError( - f"{label} reference_registry entry {name!r} identity fields " - f"must be exactly {sorted(expected_identity_fields)}, got " - f"{sorted(identity)}." - ) - records[name] = UKInputMassReferenceDescriptor( - name=name, - filename=identity["filename"], - revision=identity["revision"], - sha256=identity["sha256"], - vintage=identity["vintage"], - totals_sha256=entry["totals_sha256"], - scope_note=entry["scope_note"], + records[name] = UKCurrentBuildReference( + name=name, scope_note=entry["scope_note"] ) return dict(sorted(records.items())) @@ -794,49 +825,61 @@ def uk_default_input_mass_reviewed_exclusions() -> Mapping[ def load_uk_input_mass_reference(source: str | Path) -> UKInputMassReference: - """Load frozen reference totals emitted by the #609 measurement tooling. - - Schema: ``{"schema_version": 1, "identity": {filename, revision, sha256, - vintage}, "totals": {column: weighted_total}}`` — totals keys are flat - frame column names, matching :func:`uk_input_mass_totals`. The identity - names the pinned artifact the totals were measured from and is recorded - verbatim in the gate details and the attestation evidence digest. + """Load the input-mass reference a cut wrote from its reference build. + + Schema 2: ``{"schema_version": 2, "kind": "uk_input_mass_reference", + "label": "@", "identity": {filename, revision, sha256, + vintage}, "totals": {column: weighted_total}, "carriers": {column: + distinct source households}}``. The revision is the build, the totals + and carriers are :func:`uk_input_mass_totals` and + :func:`uk_input_mass_carriers` of the build's export frame at its own + solved weights (:func:`uk_input_mass_reference_payload`). The file stays + beside the cut's reference: its totals are licensed-data aggregates. """ path = Path(source) payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, Mapping): raise ValueError(f"{path}: input-mass reference must be a JSON object.") - if payload.get("schema_version") != 1: + if payload.get("schema_version") != UK_INPUT_MASS_REFERENCE_SCHEMA_VERSION: raise ValueError( - f"{path}: input-mass reference schema_version must be 1, got " - f"{payload.get('schema_version')!r}." + f"{path}: input-mass reference schema_version must be " + f"{UK_INPUT_MASS_REFERENCE_SCHEMA_VERSION}, got " + f"{payload.get('schema_version')!r}; the enhanced-FRS totals retired " + "with microcosm#1095, so write the reference from the line's current " + "build (tools/resolve_uk_reference.py)." + ) + if payload.get("kind") != UK_INPUT_MASS_REFERENCE_KIND: + raise ValueError( + f"{path}: input-mass reference kind must be {UK_INPUT_MASS_REFERENCE_KIND!r}." ) identity = payload.get("identity") if not isinstance(identity, Mapping): raise ValueError(f"{path}: input-mass reference needs an 'identity' object.") totals = payload.get("totals") - if not isinstance(totals, Mapping): - raise ValueError(f"{path}: input-mass reference needs a 'totals' object.") - reference = UKInputMassReference( + carriers = payload.get("carriers") + if not isinstance(totals, Mapping) or not isinstance(carriers, Mapping): + raise ValueError( + f"{path}: input-mass reference needs 'totals' and 'carriers' objects." + ) + label = payload.get("label") + if not isinstance(label, str) or label.count("@") != 1: + raise ValueError( + f"{path}: input-mass reference label must be @." + ) + if label.split("@", 1)[1] != identity.get("revision"): + raise ValueError( + f"{path}: input-mass reference label {label!r} does not name the " + f"build its identity records ({identity.get('revision')!r})." + ) + return UKInputMassReference( totals=dict(totals), filename=str(identity.get("filename", "")), revision=str(identity.get("revision", "")), sha256=str(identity.get("sha256", "")), vintage=str(identity.get("vintage", "")), - ) - observed_digest = _input_mass_reference_evidence_sha256(reference) - for descriptor in UK_INPUT_MASS_REFERENCE_REGISTRY.values(): - if ( - reference.identity == descriptor.identity - and observed_digest == descriptor.totals_sha256 - ): - return reference - known = sorted(UK_INPUT_MASS_REFERENCE_REGISTRY) - raise ValueError( - f"{path}: input-mass reference sidecar did not match any reviewed " - f"reference {known}; observed identity {reference.identity} with " - f"canonical evidence sha256 {observed_digest}." + label=label, + carriers=dict(carriers), ) @@ -856,8 +899,22 @@ class UKInputMassReference: revision: str sha256: str vintage: str + label: str | None = None + carriers: Mapping[str, int] | None = None def __post_init__(self) -> None: + if self.carriers is not None: + carriers: dict[str, int] = {} + for name, count in self.carriers.items(): + if type(count) is not int or count < 0: + raise ValueError( + "UK input-mass reference carriers must be non-negative " + f"integers; got {name!r} -> {count!r}." + ) + carriers[str(name)] = count + object.__setattr__( + self, "carriers", MappingProxyType(dict(sorted(carriers.items()))) + ) if not isinstance(self.totals, Mapping) or not self.totals: raise ValueError( "UK input-mass reference totals must be a non-empty mapping." @@ -918,12 +975,6 @@ def _input_mass_reference_evidence_sha256( return hashlib.sha256(canonical).hexdigest() -def _validate_input_mass_reference(reference: UKInputMassReference) -> None: - _validate_input_mass_reference_for_descriptor( - reference, _UK_INPUT_MASS_REFERENCE_DESCRIPTOR - ) - - def _validate_input_mass_reference_for_descriptor( reference: UKInputMassReference, descriptor: UKInputMassReferenceDescriptor, @@ -935,8 +986,7 @@ def _validate_input_mass_reference_for_descriptor( raise ValueError( "UK input-mass reference identity must match the reviewed " f"{descriptor.name}; expected {expected_identity}, got " - f"{reference.identity}. Known references: " - f"{sorted(UK_INPUT_MASS_REFERENCE_REGISTRY)}." + f"{reference.identity}." ) observed_digest = _input_mass_reference_evidence_sha256(reference) if observed_digest != descriptor.totals_sha256: @@ -1020,6 +1070,161 @@ def uk_input_mass_totals( return totals +def uk_input_mass_carriers( + frame: Frame, + *, + columns: Iterable[str] | None = None, +) -> dict[str, int]: + """Distinct source households carrying each column :func:`uk_input_mass_totals` sums. + + A column's carriers are the distinct ``household_source_id`` values among + rows of positive weight whose value is nonzero (a boolean: ``True``), + counted as the stage receipts count support + (:func:`~microcosm.build.uk_runtime.receipt_counts.distinct_count`): + support copies and clones repeat their source household, so they count + once. A person or benefit unit row counts its household's source. A + figure resting on fewer than ``UK_DISCLOSURE_FLOOR`` carriers is withheld + from what a report prints (microcosm#1095). + """ + + from microcosm.build.uk_runtime.receipt_counts import ( + distinct_count, + source_household_ids, + ) + + schema = frame.schema + household = frame.table("household") + household_sources = pd.Series( + source_household_ids(household, stage="input-mass carriers"), + index=household[schema.id_column("household")].to_numpy(), + ) + person = frame.table("person") + person_sources = household_sources.reindex( + person[schema.membership_column("household")].to_numpy() + ).to_numpy() + benunit_sources = None + if "benunit" in frame.entities: + first_member = ( + pd.DataFrame( + { + "benunit": person[schema.membership_column("benunit")].to_numpy(), + "source": person_sources, + } + ) + .drop_duplicates("benunit") + .set_index("benunit")["source"] + ) + benunit_sources = first_member.reindex( + frame.table("benunit")[schema.id_column("benunit")].to_numpy() + ).to_numpy() + sources_by_entity = { + "household": household_sources.to_numpy(), + "person": person_sources, + "benunit": benunit_sources, + } + structural = {schema.person_id_column, *PERSON_REFERENCE_COLUMNS} + for group in schema.group_entities: + structural.add(schema.id_column(group)) + structural.add(schema.membership_column(group)) + requested = None if columns is None else {str(name) for name in columns} + carriers: dict[str, int] = {} + for entity in frame.entities: + sources = sources_by_entity.get(entity) + if sources is None: + continue + table = frame.table(entity) + weights = np.asarray(frame.resolve_weights(entity).values, dtype=np.float64) + for column in table.columns: + if column in structural or column in UK_EXPORTED_WEIGHT_COLUMNS: + continue + if requested is not None and column not in requested: + continue + values = table[column] + if pd.api.types.is_bool_dtype(values): + nonzero = values.fillna(False).to_numpy(dtype=bool) + elif pd.api.types.is_numeric_dtype(values): + numeric = pd.to_numeric(values, errors="coerce").fillna(0.0) + nonzero = numeric.to_numpy(dtype=np.float64) != 0.0 + else: + continue + carriers[column] = distinct_count(sources, nonzero, weights) + return carriers + + +def uk_input_mass_reference_payload( + h5_path: str | Path, + *, + label: str, + filename: str, + revision: str, + sha256: str, + vintage: str = "2024_25", +) -> dict[str, object]: + """The schema-2 input-mass reference a cut writes from its reference build. + + Totals and carriers come from the build's export frame at its own solved + weights (:func:`~.weight_year.uk_recorded_solve_year_frame`), so a build + an earlier policyengine-uk wrote compares on the weights it was solved + with, as the candidate does. + """ + + from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + uk_release_export_frame, + ) + from microcosm.build.uk_runtime.weight_year import uk_recorded_solve_year_frame + + frame, _provenance = load_uk_national_frame(h5_path) + export = uk_release_export_frame(uk_recorded_solve_year_frame(frame)) + return { + "schema_version": UK_INPUT_MASS_REFERENCE_SCHEMA_VERSION, + "kind": UK_INPUT_MASS_REFERENCE_KIND, + "label": label, + "identity": { + "filename": filename, + "revision": revision, + "sha256": sha256, + "vintage": vintage, + }, + "totals": uk_input_mass_totals(export), + "carriers": uk_input_mass_carriers(export), + } + + +def _thin_columns( + candidate_carriers: Mapping[str, int] | None, + reference_carriers: Mapping[str, int] | None, +) -> frozenset[str]: + """Columns one or two source households carry on either side.""" + + from microcosm.build.uk_runtime.receipt_counts import UK_DISCLOSURE_FLOOR + + if candidate_carriers is None or reference_carriers is None: + return frozenset() + return frozenset( + column + for column in set(candidate_carriers) | set(reference_carriers) + if any( + 0 < count < UK_DISCLOSURE_FLOOR + for count in ( + candidate_carriers.get(column), + reference_carriers.get(column), + ) + if count is not None + ) + ) + + +def _withheld_failure(message: str, thin: frozenset[str]) -> str: + column = message.split(":", 1)[0] + if column not in thin: + return message + return ( + f"{column}: its mass check failed on a column one or two source " + "households carry on one side; the figures are withheld." + ) + + def uk_input_mass_parity_gate( candidate_totals: Mapping[str, float], reference: UKInputMassReference, @@ -1028,8 +1233,9 @@ def uk_input_mass_parity_gate( policy: UKInputMassParityPolicy, candidate_name: str = "uk_release_candidate", now: date | None = None, + candidate_carriers: Mapping[str, int] | None = None, ) -> GateResult: - """Require persisted UK input mass to survive against a frozen reference. + """Require persisted UK input mass to survive against the reference. Wraps :func:`microcosm.build.gates.input_mass_parity_gate` verbatim — a zero candidate total fails at any tolerance (the #278 signature), @@ -1039,6 +1245,12 @@ def uk_input_mass_parity_gate( within tolerance is stale and **fails**; an exclusion outside the audited surface (absent from the reference, or below the reference floor) is dormant and reported. + + Given both sides' carriers (``candidate_carriers`` and the reference's + own), a column one or two source households carry on either side is still + checked, but its figures never reach the report: its failure reads by + name, and it leaves ``worst_drifts`` for ``withheld_thin_columns`` + (UKDS disclosure control, microcosm#1095). """ if not isinstance(reference, UKInputMassReference): @@ -1085,7 +1297,8 @@ def uk_input_mass_parity_gate( if probe.passed: stale.append(column) - failures = list(base.failures) + thin = _thin_columns(candidate_carriers, reference.carriers) + failures = [_withheld_failure(message, thin) for message in base.failures] if stale: failures.append( "Stale reviewed input-mass exclusions — the column is within " @@ -1105,6 +1318,14 @@ def uk_input_mass_parity_gate( failures=tuple(failures), details={ **dict(base.details), + "worst_drifts": { + name: drift + for name, drift in dict(base.details["worst_drifts"]).items() + if name not in thin + }, + "withheld_thin_columns": sorted( + thin & (set(reference.totals) | set(candidate_totals)) + ), "stale_exclusions": stale, "dormant_exclusions": dormant, "expired_exclusions": expired, @@ -1113,6 +1334,7 @@ def uk_input_mass_parity_gate( "reference": descriptor.name, "reference_scope_note": descriptor.scope_note, "reference_identity": reference.identity, + "reference_label": reference.label, }, ) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_efrs_weighted_totals.py b/packages/microcosm-build/tests/engine/uk/test_uk_efrs_weighted_totals.py index 6e5a2dc203..f7478b5cd4 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_efrs_weighted_totals.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_efrs_weighted_totals.py @@ -1,7 +1,8 @@ """Weighted-totals extraction from the pinned enhanced-FRS incumbent (#609). -This is the reference side of the UK input_mass_parity gate: the frozen totals -a candidate is measured against. The licensed artifact is never in CI, so the +This was the reference side of the UK input_mass_parity gate until +microcosm#1095 moved the gate to the current build of the cut's own line; the +extraction stays as the measurement of the frozen enhanced-FRS reference. The licensed artifact is never in CI, so the sha pin is stubbed and the extraction runs against a synthetic H5 with the same shape — enough to prove the plumbing (table reading, engine-variable classification, weight broadcast, key format) before a licensed run pays for it. @@ -139,59 +140,6 @@ def test_payload_pins_the_reference_identity_the_gate_records( } -def test_payload_loads_as_a_gate_reference( - synthetic_efrs, - tmp_path, - monkeypatch, -) -> None: - """The emitted file is exactly what the gate's loader accepts.""" - - from microcosm.build.uk_runtime import weighted_integrity - from microcosm.build.uk_runtime.weighted_integrity import ( - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - UK_INPUT_MASS_REFERENCE_REGISTRY, - UKInputMassParityPolicy, - load_uk_input_mass_reference, - uk_input_mass_parity_gate, - ) - - payload = TOOL.build_weighted_totals(synthetic_efrs) - path = tmp_path / "efrs_weighted_totals.json" - path.write_text(json.dumps(payload), encoding="utf-8") - - # This synthetic source deliberately stubs the licensed artifact sha. - # The reviewed 131-column digest is covered by the runtime regressions. - monkeypatch.setattr( - weighted_integrity, - "_input_mass_reference_evidence_sha256", - lambda _reference: UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - ) - descriptor = UK_INPUT_MASS_REFERENCE_REGISTRY["efrs-post-calibration"] - reference = load_uk_input_mass_reference(path) - identical = uk_input_mass_parity_gate( - dict(reference.totals), - reference, - descriptor=descriptor, - policy=UKInputMassParityPolicy( - relative_tolerance=0.0, - minimum_reference_total=0.0, - ), - ) - zeroed = uk_input_mass_parity_gate( - {name: 0.0 for name in reference.totals}, - reference, - descriptor=descriptor, - policy=UKInputMassParityPolicy( - relative_tolerance=0.0, - minimum_reference_total=0.0, - ), - ) - - assert reference.filename == TOOL.SOURCE_FILENAME - assert identical.passed - assert not zeroed.passed - - def test_emitting_totals_never_rewrites_the_committed_reference( synthetic_efrs, tmp_path, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index ee3a33e071..bd8f4369c8 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -1835,7 +1835,10 @@ def test_thresholds_match_the_schema4_manifest(self, manifest) -> None: params["uk_weight_ratio"]["maximum_max_to_median_ratio"] == 1_151.2542195939373 ) - assert params["uk_input_mass_parity"]["relative_tolerance"] == 4.521811483823806 + # The a6299504 -> eeb6093e pair's exact maximum (microcosm#1095). + assert ( + params["uk_input_mass_parity"]["relative_tolerance"] == 5.9984285545209834 + ) assert params["uk_input_mass_parity"]["minimum_reference_total"] == 0.0 assert ( params["uk_target_fit"]["max_abs_relative_error"] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py index 90e0711609..1a79c38a1f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_gate_battery_contract_pins.py @@ -31,9 +31,7 @@ from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY from microcosm.build.uk_runtime.weighted_integrity import ( UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE, - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, UK_INPUT_MASS_REFERENCE_REGISTRY, - uk_default_input_mass_reviewed_exclusions, ) from microcosm.data import contract as data_contract from test_support.paths import paths_for @@ -165,26 +163,12 @@ def test_degenerate_evidence_pin_mirrors_the_committed_register(self) -> None: _canonical_sha256(payload) ) - def test_input_mass_evidence_pin_mirrors_the_wrapped_reference(self) -> None: - from microcosm.build.gate_battery import _canonical_sha256 + def test_the_input_mass_evidence_is_not_pinned(self) -> None: + """Its digest covers the reference build's own totals, which change + with every cut (microcosm#1095); the report names the build.""" - reference = "efrs-post-calibration" - records = uk_default_input_mass_reviewed_exclusions()[reference] - assert data_contract._UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 == ( - _canonical_sha256( - { - "reference": reference, - "reference_evidence_sha256": ( - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 - ), - "exclusions_policy": "committed", - "reviewed_exclusions": { - name: record.policy_payload() - for name, record in sorted(records.items()) - }, - } - ) - ) + assert not hasattr(data_contract, "_UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256") + assert not hasattr(data_contract, "_UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256") def test_numeric_threshold_mirrors_match_the_committed_spec(self) -> None: spec = load_country_spec("uk") @@ -205,14 +189,7 @@ def test_numeric_threshold_mirrors_match_the_committed_spec(self) -> None: == (params["uk_target_fit"]["max_abs_relative_error"]) ) assert data_contract._UK_INPUT_MASS_ACTIVE_REFERENCE == reference - assert ( - data_contract._UK_INPUT_MASS_REFERENCE_IDENTITY - == input_mass["reference_registry"][reference]["identity"] - ) - assert ( - data_contract._UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 - == (input_mass["reference_registry"][reference]["totals_sha256"]) - ) + assert input_mass["reference_registry"][reference]["kind"] == "current_build" assert input_mass["reviewed_exclusions_resource"] == ( UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py b/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py index 9b19c3fc2f..e3deb3f7d2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_resolve_uk_reference.py @@ -13,6 +13,7 @@ UK_NATIONAL_RELEASE, UKReferenceHub, sha256_of, + stub_reference_evidence, ) from tools import resolve_uk_reference as tool @@ -41,7 +42,9 @@ def test_output_dir_keeps_the_dataset_and_its_record(hub, tmp_path: Path) -> Non assert ( tool.main( - ["--release", UK_DENSE_RELEASE, "--output-dir", str(out)], hub_download=hub + ["--release", UK_DENSE_RELEASE, "--output-dir", str(out)], + hub_download=hub, + write_evidence=stub_reference_evidence, ) == 0 ) @@ -50,6 +53,7 @@ def test_output_dir_keeps_the_dataset_and_its_record(hub, tmp_path: Path) -> Non assert reference.build == DENSE_BUILD assert (out / reference.filename).read_bytes() == b"dense" assert reference.sha256 == sha256_of(b"dense") + assert (out / "input_mass_reference.json").exists() def test_a_directory_that_holds_a_reference_is_refused( @@ -57,9 +61,10 @@ def test_a_directory_that_holds_a_reference_is_refused( ) -> None: out = tmp_path / "reference" args = ["--release", UK_NATIONAL_RELEASE, "--output-dir", str(out)] - assert tool.main(args, hub_download=hub) == 0 + stub = stub_reference_evidence + assert tool.main(args, hub_download=hub, write_evidence=stub) == 0 - assert tool.main(args, hub_download=hub) == 1 + assert tool.main(args, hub_download=hub, write_evidence=stub) == 1 assert "keeps the reference it started with" in capsys.readouterr().err diff --git a/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py index 18192048d9..9e809d7a43 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_run_uk_cut.py @@ -44,7 +44,6 @@ def test_every_declared_input_resolves_to_a_committed_pin(): "aps_2024_25", "hmrc_personal_incomes_ods", "chronicle_uk", - "input_mass_reference", } for source in sources.values(): assert source.files @@ -122,18 +121,6 @@ def test_an_unknown_pin_selector_is_refused(tmp_path): cut.load_input_sources(path) -def test_the_hosted_mass_reference_is_the_one_the_gate_binds(): - from microcosm.build.uk_runtime.weighted_integrity import ( - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - ) - - manifest = json.loads(cut.CUT_INPUTS.read_text()) - pin = manifest["file_pins"]["uk_input_mass_reference.json"] - assert pin["canonical_evidence_sha256"] == UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 - source = next(s for s in manifest["sources"] if s["key"] == "input_mass_reference") - assert source["path_in_repo"].endswith(UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256) - - def test_every_spine_tab_flag_has_a_pinned_input(): """The spine's licensed tabs are all fetched, under the names it reads.""" @@ -213,9 +200,14 @@ def _reference(paths: cut.CutPaths, *, build: str = REFERENCE_BUILD, data=b"ref" }, resolved_at="2026-11-10T12:00:00Z", ) + from test_support.microcosm_build.uk_cut_reference import stub_reference_evidence + paths.reference_dir.mkdir(parents=True, exist_ok=True) paths.reference_h5(reference).write_bytes(data) write_uk_cut_reference(paths.reference_record, reference) + stub_reference_evidence( + reference, paths.reference_h5(reference), paths.reference_dir + ) return reference @@ -1257,6 +1249,12 @@ def start(): # --------------------------------------------------------------------------- # +def _stub_evidence(reference, h5_path, output_dir): + from test_support.microcosm_build.uk_cut_reference import stub_reference_evidence + + return stub_reference_evidence(reference, h5_path, output_dir) + + def _reference_hub(tmp_path): from test_support.microcosm_build.uk_cut_reference import ( UK_NATIONAL_RELEASE, @@ -1272,7 +1270,9 @@ def test_the_reference_is_resolved_once_and_kept(tmp_path): runner, _ = _runner(tmp_path, reference=False) hub = _reference_hub(tmp_path) - reference = cut.run_reference(runner, hub_download=hub) + reference = cut.run_reference( + runner, hub_download=hub, write_evidence=_stub_evidence + ) assert reference.build == REFERENCE_BUILD assert runner.paths.reference_h5(reference).read_bytes() == b"current build" @@ -1282,7 +1282,10 @@ def test_the_reference_is_resolved_once_and_kept(tmp_path): # A promotion after the fetch moves nothing in this cut. hub.publish(cut.RELEASE_ID, "uk-frs-calibration-attempt-later", dataset=b"newer") calls = len(hub.calls) - assert cut.run_reference(runner, hub_download=hub).build == REFERENCE_BUILD + assert ( + cut.run_reference(runner, hub_download=hub, write_evidence=_stub_evidence).build + == REFERENCE_BUILD + ) assert len(hub.calls) == calls @@ -1291,10 +1294,12 @@ def test_re_resolving_sets_the_reference_aside_and_moves_to_the_current_build( ): runner, _ = _runner(tmp_path, reference=False) hub = _reference_hub(tmp_path) - cut.run_reference(runner, hub_download=hub) + cut.run_reference(runner, hub_download=hub, write_evidence=_stub_evidence) hub.publish(cut.RELEASE_ID, "uk-frs-calibration-attempt-later", dataset=b"newer") - reference = cut.run_reference(runner, re_resolve=True, hub_download=hub) + reference = cut.run_reference( + runner, re_resolve=True, hub_download=hub, write_evidence=_stub_evidence + ) assert reference.build == "uk-frs-calibration-attempt-later" kept = list(runner.paths.cut.glob("reference.superseded-*")) @@ -1305,7 +1310,9 @@ def test_an_unusable_reference_is_never_resolved_over(tmp_path): runner, _ = _runner(tmp_path) runner.paths.reference_h5(cut.cut_reference(runner.paths)).write_bytes(b"x") with pytest.raises(cut.CutError, match="--re-resolve-reference"): - cut.run_reference(runner, hub_download=_reference_hub(tmp_path)) + cut.run_reference( + runner, hub_download=_reference_hub(tmp_path), write_evidence=_stub_evidence + ) def test_a_hub_without_a_certified_current_build_stops_the_cut(tmp_path): @@ -1320,7 +1327,7 @@ def test_a_hub_without_a_certified_current_build_stops_the_cut(tmp_path): evidence={"kind": "uk_release_certification", "shippable": False}, ) with pytest.raises(cut.CutError, match="no reference to compare with"): - cut.run_reference(runner, hub_download=hub) + cut.run_reference(runner, hub_download=hub, write_evidence=_stub_evidence) state = json.loads(runner.paths.state.read_text())["steps"]["reference"] assert state["status"] == "failed" @@ -1379,3 +1386,36 @@ def test_the_receipt_records_the_reference(tmp_path): receipt = json.loads(runner.paths.receipt.read_text()) assert receipt["reference"]["build"] == REFERENCE_BUILD assert "incumbent_efrs" not in receipt["inputs"] + + +def test_the_cut_takes_its_input_mass_reference_from_its_own_reference(tmp_path): + """The hosted enhanced-FRS totals are gone; certify reads the totals the + reference step wrote from the cut's reference build (microcosm#1095).""" + + manifest = json.loads(cut.CUT_INPUTS.read_text()) + assert "input_mass_reference" not in {s["key"] for s in manifest["sources"]} + assert "file_pins" not in manifest + runner, _ = _runner(tmp_path) + argv = cut.certify_argv( + runner.paths, + python="py", + candidate_sha256="d" * 64, + spine_sha256="e" * 64, + sources=runner.sources, + ) + assert argv[argv.index("--input-mass-reference") + 1] == str( + runner.paths.reference_input_mass + ) + + +def test_a_reference_without_its_evidence_gets_it_written_not_re_resolved(tmp_path): + runner, _ = _runner(tmp_path) + runner.paths.reference_input_mass.unlink() + assert not cut.reference_done(runner.paths)[0] + + reference = cut.run_reference( + runner, hub_download=None, write_evidence=_stub_evidence + ) + + assert reference.build == REFERENCE_BUILD + assert cut.reference_done(runner.paths)[0] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py index 242d5d872d..3bbdad4245 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py @@ -195,6 +195,9 @@ def _tables(*, n: int = 4, weights=None): { "household_id": household_ids, "household_weight": np.asarray(weights, dtype=float), + # Each household its own survey source, as the input-mass gate's + # carriers read them (microcosm#1095). + "household_source_id": household_ids + 1000, "household_is_spi_synthetic": np.arange(n) % 2 == 1, "household_is_capital_gains_clone": np.arange(n) % 4 >= 2, "owned_land": land, @@ -227,13 +230,20 @@ def _parity(**overrides) -> SimpleNamespace: return SimpleNamespace(**fields) -def _reference() -> UKInputMassReference: +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +REFERENCE_LABEL = f"microcosm-uk-2024-25-national@{REFERENCE_BUILD}" + + +def _reference(totals=None) -> UKInputMassReference: + """The current build's input-mass reference (microcosm#1095).""" + return UKInputMassReference( - totals={"employment_income": 10.0}, - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", + totals={"employment_income": 10.0} if totals is None else totals, + filename="microcosm_uk_2024_25.h5", + revision=REFERENCE_BUILD, + sha256="9" * 64, + vintage="2024_25", + label=REFERENCE_LABEL, ) @@ -1088,20 +1098,13 @@ def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: assert "uk_soc2020_sub_major_group_enum_domain" in passed def test_input_mass_parity_compares_the_release_surface(self) -> None: - # The frame carries the spine's WAS property_wealth and the reference - # stores its own total within tolerance. Compared on the raw frame the - # signed exclusion would be stale and the gate would fail; the gate - # compares the export frame, where the column is absent, so the - # exclusion is in force and nothing is stale (microcosm#1106). + # The frame carries the spine's WAS property_wealth, which the release + # leaves to the engine (microcosm#1106); the current build's export + # does not carry it either. The gate compares the export frame, so the + # spine's copy never enters the comparison as a candidate-only column. person, benunit, household = _tables() household = household.assign(property_wealth=100.0) - reference = UKInputMassReference( - totals={"employment_income": 10.0, "property_wealth": 400.0}, - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", - ) + reference = _reference() battery = _run_battery( (person, benunit, household), parity=_parity(), @@ -1112,7 +1115,8 @@ def test_input_mass_parity_compares_the_release_surface(self) -> None: assert outcome.status is GateStatus.PASSED, outcome.result.failures details = outcome.result.details assert details["stale_exclusions"] == [] - assert "property_wealth" in details["reviewed_exclusions"] + assert "property_wealth" not in details["candidate_only_columns"] + assert details["reference_label"] == REFERENCE_LABEL def test_empty_fit_records_fail_closed(self) -> None: # Present-but-empty is not absent: a fit stage that ran and emitted @@ -1255,11 +1259,11 @@ def test_every_exclusion_gate_shares_the_injected_clock(self) -> None: def test_an_expired_register_fails_closed(self) -> None: # The committed input-mass register carries no entry since the - # 2026-09-21 retirements, so the expiry discipline is exercised on an - # injected receipt that lapses before the clock below; the other two - # registers still expire from their committed entries. + # enhanced-FRS reference retired (microcosm#1095), so the expiry + # discipline is exercised on an injected receipt, signed for the + # reference build, that lapses before the clock below. lapsed = { - "efrs-post-calibration": { + REFERENCE_LABEL: { "owned_land": { "reason": "injected: a reviewed exclusion whose window has closed", "approved_by": "juaristi22", @@ -2070,14 +2074,7 @@ def test_drifted_declared_pin_fails_closed(self, uk_gates) -> None: entry = {e.id: e for e in uk_gates.gates}["uk_input_mass_parity"] drifted = dict(entry.parameters) drifted["reference_registry"] = { - name: ( - { - **payload, - "totals_sha256": "0" * 64, - } - if name == "efrs-post-calibration" - else payload - ) + name: {**payload, "scope_note": "a drifted scope note"} for name, payload in drifted["reference_registry"].items() } binding = UK_GATE_REGISTRY["input_mass_parity"] @@ -2823,3 +2820,50 @@ def test_bound_despite_floor_approvals_are_window_checked_on_every_line( "bound_despite_support_floor/census_households" ] assert result.details["out_of_line_exclusions"] == ["local_authority/LA_B"] + + +class TestInputMassReferenceBuild: + """The exclusions a gate applies are the ones signed for its reference build.""" + + def test_an_exclusion_signed_for_another_build_is_dormant(self) -> None: + signed = { + "microcosm-uk-2024-25-national@uk-frs-calibration-attempt-older": { + "employment_income": { + "reason": "injected: signed for an earlier reference build", + "approved_by": "juaristi22", + "adjudication": "microcosm#1095 (test)", + "approved_on": "2026-11-01", + "expires_on": "2027-03-01", + } + } + } + person, benunit, household = _tables() + battery = _run_battery( + (person, benunit, household), + parity=_parity(), + fit_records=(FitWeightRecord("spi_qrf", "importance"),), + extra_artifacts={ + # The candidate carries far more than the reference: past the + # tolerance, which an exclusion for this build would excuse. + "input_mass_reference": _reference({"employment_income": 1.0e-3}), + "reviewed_input_mass_exclusions": signed, + }, + ) + outcome = {o.entry.id: o for o in battery.outcomes}["uk_input_mass_parity"] + # The entry is for another build, so the drift fails here. + assert outcome.status is GateStatus.FAILED + assert outcome.result.details["reviewed_exclusions"] == {} + + def test_an_exclusion_key_that_names_no_build_is_refused(self) -> None: + person, benunit, household = _tables() + battery = _run_battery( + (person, benunit, household), + parity=_parity(), + fit_records=(FitWeightRecord("spi_qrf", "importance"),), + extra_artifacts={ + "reviewed_input_mass_exclusions": {"efrs-post-calibration": {}}, + }, + ) + outcome = {o.entry.id: o for o in battery.outcomes}["uk_input_mass_parity"] + assert outcome.status is GateStatus.FAILED + assert "@" in " ".join(outcome.result.failures) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_signoff_draft.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_signoff_draft.py index 8d29d47c1b..4fb0995312 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_signoff_draft.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_dense_signoff_draft.py @@ -1382,7 +1382,11 @@ def test_the_national_graph_review_invariant_is_kept( def test_the_review_registers_load_from_the_package(): registers = REAL_REVIEW_REGISTERS() assert len(registers) == 4 - assert all(registers) + # The input-mass register (the second) is empty since the enhanced-FRS + # reference retired (microcosm#1095); its entries are signed per build. + degenerate, input_mass, exclusions, floor = registers + assert degenerate and exclusions and floor + assert input_mass == {} def test_out_inside_the_repository_is_refused(build_1, repo, capsys): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py index 1210ccd451..1fd76fb92f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py @@ -29,7 +29,7 @@ uk_zero_weight_strata_gate, ) from microcosm.build.uk_runtime.weighted_integrity import ( - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, + UK_CURRENT_BUILD_REFERENCE, UKInputMassReferenceDescriptor, UKReviewedExclusion, ) @@ -865,26 +865,23 @@ def test_ported_june_parity_gates_reject_empty_evidence() -> None: assert "evidence is empty" in " ".join(fit.failures) -def _input_mass_reference(totals=None) -> UKInputMassReference: +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" + + +def _input_mass_reference(totals=None, carriers=None) -> UKInputMassReference: return UKInputMassReference( totals=({"employment_income": 10.0} if totals is None else totals), - filename="enhanced_frs_2024_25.h5", - revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - sha256=("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), + filename="microcosm_uk_2024_25.h5", + revision=REFERENCE_BUILD, + sha256="9" * 64, vintage="2024_25", + label=f"microcosm-uk-2024-25-national@{REFERENCE_BUILD}", + carriers=carriers, ) def _input_mass_descriptor() -> UKInputMassReferenceDescriptor: - return UKInputMassReferenceDescriptor( - name="efrs-post-calibration", - filename="enhanced_frs_2024_25.h5", - revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - sha256="e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", - vintage="2024_25", - totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - scope_note="Seeded scoped-reference note.", - ) + return UK_CURRENT_BUILD_REFERENCE.descriptor_for(_input_mass_reference()) def _input_mass_policy(**overrides) -> UKInputMassParityPolicy: @@ -893,13 +890,16 @@ def _input_mass_policy(**overrides) -> UKInputMassParityPolicy: return UKInputMassParityPolicy(**fields) -def _input_mass_gate(candidate_totals, reference=None, *, policy=None): +def _input_mass_gate( + candidate_totals, reference=None, *, policy=None, candidate_carriers=None +): with patch(VALIDATE_REFERENCE, return_value=None): return uk_input_mass_parity_gate( candidate_totals, _input_mass_reference() if reference is None else reference, descriptor=_input_mass_descriptor(), policy=_input_mass_policy() if policy is None else policy, + candidate_carriers=candidate_carriers, ) @@ -910,7 +910,10 @@ def test_zeroed_input_column_fails_by_name() -> None: assert gate.name == "input_mass_parity" assert "employment_income" in gate.failures[0] assert "mass is zero" in gate.failures[0] - assert gate.details["reference_identity"]["filename"] == ("enhanced_frs_2024_25.h5") + assert gate.details["reference_identity"]["filename"] == "microcosm_uk_2024_25.h5" + assert gate.details["reference_label"] == ( + f"microcosm-uk-2024-25-national@{REFERENCE_BUILD}" + ) def test_999_permille_mass_loss_fails_by_name() -> None: @@ -935,6 +938,40 @@ def test_stale_weighted_integrity_exclusions_fail() -> None: assert "Stale reviewed input-mass exclusions" in gate.failures[0] +def test_a_thin_column_is_checked_but_its_figures_are_withheld() -> None: + """One or two source households on a side: the verdict stands, the + figures never reach the report (microcosm#1095).""" + + reference = _input_mass_reference( + totals={"employment_income": 10.0, "rare_flag": 4.0}, + carriers={"employment_income": 900, "rare_flag": 2}, + ) + gate = _input_mass_gate( + {"employment_income": 30.0, "rare_flag": 0.0}, + reference, + candidate_carriers={"employment_income": 950, "rare_flag": 0}, + ) + + assert not gate.passed + rare = [failure for failure in gate.failures if failure.startswith("rare_flag:")] + assert rare == [ + "rare_flag: its mass check failed on a column one or two source " + "households carry on one side; the figures are withheld." + ] + employment = [f for f in gate.failures if f.startswith("employment_income:")] + assert employment and "30" in employment[0] + assert "rare_flag" not in gate.details["worst_drifts"] + assert "employment_income" in gate.details["worst_drifts"] + assert gate.details["withheld_thin_columns"] == ["rare_flag"] + + +def test_without_carriers_nothing_is_withheld() -> None: + gate = _input_mass_gate({"employment_income": 0.0}) + + assert gate.details["withheld_thin_columns"] == [] + assert "mass is zero" in gate.failures[0] + + def test_weighted_integrity_type_errors_are_named() -> None: with pytest.raises(TypeError, match="reference must be UKInputMassReference"): uk_input_mass_parity_gate( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_weighted_integrity.py index 9b2be23958..0fe1b2ea20 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_weighted_integrity.py @@ -25,8 +25,8 @@ SPI_INCOME_QRF_OUTPUT_COLUMNS, ) from microcosm.build.uk_runtime.weighted_integrity import ( + UK_CURRENT_BUILD_REFERENCE, UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE, - UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, UK_INPUT_MASS_REFERENCE_REGISTRY, UKInputMassParityPolicy, UKInputMassReference, @@ -39,6 +39,7 @@ load_uk_reference_scoped_exclusion_register, load_uk_reviewed_exclusion_register, uk_default_input_mass_reviewed_exclusions, + uk_input_mass_carriers, uk_input_mass_parity_gate, uk_input_mass_totals, uk_qrf_tail_concentration_columns, @@ -94,12 +95,17 @@ def _frame( ) +REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +REFERENCE_LABEL = f"microcosm-uk-2024-25-national@{REFERENCE_BUILD}" + + def _reference(totals, **overrides) -> UKInputMassReference: fields = { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": ("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), + "filename": "microcosm_uk_2024_25.h5", + "revision": REFERENCE_BUILD, + "sha256": "9" * 64, "vintage": "2024_25", + "label": REFERENCE_LABEL, } fields.update(overrides) return UKInputMassReference(totals=totals, **fields) @@ -107,12 +113,12 @@ def _reference(totals, **overrides) -> UKInputMassReference: def _descriptor(**overrides) -> UKInputMassReferenceDescriptor: fields = { - "name": "efrs-post-calibration", - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", + "name": "current_build", + "filename": "microcosm_uk_2024_25.h5", + "revision": REFERENCE_BUILD, + "sha256": "9" * 64, "vintage": "2024_25", - "totals_sha256": UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, + "totals_sha256": "8" * 64, "scope_note": "Seeded scoped-reference note.", } fields.update(overrides) @@ -277,80 +283,34 @@ def test_input_mass_reference_identity_is_recorded() -> None: ) assert gate.details["reference_identity"] == { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": ("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), + "filename": "microcosm_uk_2024_25.h5", + "revision": REFERENCE_BUILD, + "sha256": "9" * 64, "vintage": "2024_25", } + assert gate.details["reference_label"] == REFERENCE_LABEL -def test_input_mass_reference_rejects_substituted_totals_at_approved_identity( - tmp_path, -) -> None: - caller_self_reference = _reference( - {"employment_income": 1.0}, - ) - - with pytest.raises(ValueError, match="reference totals must match the reviewed"): - uk_input_mass_parity_gate( - {"employment_income": 1.0}, - caller_self_reference, - descriptor=_descriptor(), - policy=_policy(), - ) - path = tmp_path / "self-reference.json" - path.write_text( - json.dumps( - { - "schema_version": 1, - "identity": caller_self_reference.identity, - "totals": dict(caller_self_reference.totals), - } - ), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="did not match any reviewed reference"): - load_uk_input_mass_reference(path) +def test_the_descriptor_is_made_from_the_reference_itself() -> None: + """The spec names the kind; the identity and totals digest the gate checks + are the reference build's own (microcosm#1095).""" + reference = _reference({"employment_income": 1.0}) + descriptor = UK_CURRENT_BUILD_REFERENCE.descriptor_for(reference) -def test_input_mass_reference_identity_pin_cannot_be_shadowed_from_cwd( - tmp_path, - monkeypatch, -) -> None: - shadow = { - "schema_version": 3, - "source": { - "repo_id": "caller/repo", - "repo_type": "model", - "filename": "caller.h5", - "revision": "caller", - "sha256": "b" * 64, - "url": "https://example.invalid/caller.h5", - "vintage": "caller", - "period": "2023", - "size_bytes": 1, - }, - "nonzero_shares": {"employment_income": 1.0}, - "input_entities": {"employment_income": "person"}, - } - (tmp_path / "efrs_parity_reference.json").write_text( - json.dumps(shadow), encoding="utf-8" - ) - monkeypatch.chdir(tmp_path) - caller_reference = _reference( - {"employment_income": 1.0}, - filename="caller.h5", - revision="caller", - sha256="b" * 64, - vintage="caller", + assert descriptor.identity == reference.identity + gate = uk_input_mass_parity_gate( + {"employment_income": 1.0}, reference, descriptor=descriptor, policy=_policy() ) - - with pytest.raises(ValueError, match="identity must match the reviewed"): + assert gate.passed + other = _reference({"employment_income": 2.0}) + with pytest.raises(ValueError, match="reference totals must match the reviewed"): uk_input_mass_parity_gate( - {"employment_income": 1.0}, - caller_reference, - descriptor=_descriptor(), - policy=_policy(), + {"employment_income": 1.0}, other, descriptor=descriptor, policy=_policy() + ) + with pytest.raises(ValueError, match="names no build"): + UK_CURRENT_BUILD_REFERENCE.descriptor_for( + _reference({"employment_income": 1.0}, label=None) ) @@ -409,30 +369,28 @@ def test_input_mass_policy_and_reference_validation() -> None: def test_input_mass_reference_registry_coercion_is_closed_world() -> None: - descriptor = next(iter(UK_INPUT_MASS_REFERENCE_REGISTRY.values())) - coerced = coerce_input_mass_reference_registry( - {"efrs-post-calibration": descriptor.spec_payload()}, label="fixture" + {"current_build": UK_CURRENT_BUILD_REFERENCE.spec_payload()}, label="fixture" ) assert coerced == dict(UK_INPUT_MASS_REFERENCE_REGISTRY) - extra = descriptor.spec_payload() - extra["unexpected"] = "ignored?" - with pytest.raises(ValueError, match="fields must be exactly"): - coerce_input_mass_reference_registry( - {"efrs-post-calibration": extra}, label="fixture" - ) - malformed = descriptor.spec_payload() - malformed["identity"] = {**descriptor.identity, "sha256": "A" * 64} - with pytest.raises(ValueError, match="sha256 must be a lowercase sha256"): - coerce_input_mass_reference_registry( - {"efrs-post-calibration": malformed}, label="fixture" - ) - bad_totals = descriptor.spec_payload() - bad_totals["totals_sha256"] = "not-a-sha" - with pytest.raises(ValueError, match="totals_sha256"): + extra = {**UK_CURRENT_BUILD_REFERENCE.spec_payload(), "unexpected": "x"} + with pytest.raises(ValueError, match="must be exactly"): + coerce_input_mass_reference_registry({"current_build": extra}, label="fixture") + # A reference pinned by file identity, as the enhanced FRS was, is refused. + pinned = { + "identity": { + "filename": "a.h5", + "revision": "r", + "sha256": "0" * 64, + "vintage": "v", + }, + "totals_sha256": "0" * 64, + "scope_note": "pinned", + } + with pytest.raises(ValueError, match="no longer accepted"): coerce_input_mass_reference_registry( - {"efrs-post-calibration": bad_totals}, label="fixture" + {"efrs-post-calibration": pinned}, label="fixture" ) @@ -541,42 +499,10 @@ def test_committed_exclusion_registers_load() -> None: resource=UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE, ) - assert set(input_mass) == {"efrs-post-calibration"} - # The 2026-09-21 retirements (charitable_investment_gifts and owned_land - # measured inside the fence on the first national candidate) stand; the - # certifier rehearsal on the 2026-09-30 build earned the FRS education - # input the enhanced-FRS reference barely carries (microcosm#1063 c9). - # The gate fails an entry as stale the day a build brings the column - # inside the fence, which the 2 October final build did for adult_ema - # (+74% against the reference): its entry is retired. The 3 October final - # build's certifier earned three more, signed 2026-10-04: the FRS - # access-fund outlier at the weight bound (expiring with the owed period - # rule) and the SPI channel's two legacy-benefit leaves. All three are - # retired with the microcosm#1095 ports: spi_benefit_coherence zeroes - # income-based JSA and Working Tax Credit on SPI rows, and on the - # review-head arm the columns sit at +316% and +103% of the reference and - # the access fund at +62%, its household off the weight bound, all inside - # the fence, so the gate fails them as stale. property_wealth joins with - # microcosm#1106 (signed 2026-10-11): the release leaves it to the engine, - # so the reference carries a column the candidate no longer ships. The - # QRF tail register went with its gate (microcosm#1276). - # tax_free_savings_income joins with microcosm#1360 (signed 2026-10-28): - # under the additive ISA income contract the candidate stores ISA interest - # apart, and the engine sums it into the column the reference stores. - assert set(input_mass["efrs-post-calibration"]) == { - "dfe_education_spending", - "property_wealth", - "tax_free_savings_income", - } - for record in input_mass["efrs-post-calibration"].values(): - assert record.approved_by == "juaristi22" - assert record.approved_on in {"2026-10-02", "2026-10-11", "2026-10-28"} - assert record.expires_on in {"2027-04-02", "2027-04-11", "2027-04-28"} - assert ( - "microcosm#1063" in record.adjudication - or "microcosm#1106" in record.adjudication - or "microcosm#1360" in record.adjudication - ) + # The enhanced-FRS reference's three entries (dfe_education_spending, + # property_wealth, tax_free_savings_income) retired with it + # (microcosm#1095); entries are now signed for one reference build. + assert input_mass == {} assert uk_default_input_mass_reviewed_exclusions() is ( uk_default_input_mass_reviewed_exclusions() ) @@ -921,43 +847,76 @@ def test_uk_input_mass_gate_is_the_shared_gate_plus_recorded_identity() -> None: "reference", "reference_scope_note", "reference_identity", + "reference_label", + "withheld_thin_columns", } -def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> None: - path = tmp_path / "reference.json" - path.write_text( - json.dumps( +def test_input_mass_reference_round_trips_the_cut_schema(tmp_path) -> None: + path = tmp_path / "input_mass_reference.json" + payload = { + "schema_version": 2, + "kind": "uk_input_mass_reference", + "label": REFERENCE_LABEL, + "identity": { + "filename": "microcosm_uk_2024_25.h5", + "revision": REFERENCE_BUILD, + "sha256": "9" * 64, + "vintage": "2024_25", + }, + "totals": {"employment_income": 10.5}, + "carriers": {"employment_income": 7}, + } + path.write_text(json.dumps(payload)) + + reference = load_uk_input_mass_reference(path) + + assert reference.label == REFERENCE_LABEL + assert reference.revision == REFERENCE_BUILD + assert dict(reference.totals) == {"employment_income": 10.5} + assert dict(reference.carriers) == {"employment_income": 7} + for broken, message in ( + ({"schema_version": 1}, "retired with microcosm#1095"), + ({"label": "microcosm-uk-2024-25-national@other"}, "does not name the build"), + ({"carriers": None}, "'carriers' objects"), + ): + path.write_text(json.dumps({**payload, **broken})) + with pytest.raises(ValueError, match=message): + load_uk_input_mass_reference(path) + + +def test_carriers_count_source_households_not_rows() -> None: + """Clones repeat their source household; a person counts its household's.""" + + household_ids = np.arange(1, 5, dtype=np.int64) + frame = uk_national_frame( + person=pd.DataFrame( { - "schema_version": 1, - "identity": { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": ( - "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712" - ), - "vintage": "2024_25", - }, - "totals": {"employment_income": 10.5}, + "person_id": np.arange(101, 105, dtype=np.int64), + "person_household_id": household_ids, + "person_benunit_id": np.arange(201, 205, dtype=np.int64), + "employment_income": np.arange(1, 5, dtype=float), + "rare_flag": np.array([True, True, False, False]), } - ) + ), + benunit=pd.DataFrame({"benunit_id": np.arange(201, 205, dtype=np.int64)}), + household=pd.DataFrame( + { + "household_id": household_ids, + "household_weight": np.ones(4), + "council_tax": np.full(4, 10.0), + # Two clones of one source household, and two others. + "household_source_id": np.array([7, 7, 8, 9], dtype=np.int64), + } + ), + time_period="2023", ) - # The licensed 131-column totals are intentionally unavailable to CI; - # make the small fixture present the reviewed digest while checking the - # measurement schema and registry identity resolution. - with patch.object( - weighted_integrity, - "_input_mass_reference_evidence_sha256", - return_value=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - ): - reference = load_uk_input_mass_reference(path) + carriers = uk_input_mass_carriers(frame) - assert reference.filename == "enhanced_frs_2024_25.h5" - assert dict(reference.totals) == {"employment_income": 10.5} - with pytest.raises(ValueError, match="schema_version"): - path.write_text(json.dumps({"schema_version": 9})) - load_uk_input_mass_reference(path) + assert carriers["rare_flag"] == 1 + assert carriers["employment_income"] == 3 + assert carriers["council_tax"] == 3 def test_expired_exclusion_stops_suppressing_and_names_its_receipt() -> None: diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 451577a318..ff49f5593f 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -426,25 +426,17 @@ def _is_hierarchy_diagnostics_version(value: object) -> bool: # armed. Held in lockstep with the committed spec by the build-shard sync # tests. The QRF tail-concentration gate and its pins were removed in # microcosm#1276. -_UK_INPUT_MASS_RELATIVE_TOLERANCE = 4.521811483823806 +_UK_INPUT_MASS_RELATIVE_TOLERANCE = 5.9984285545209834 _UK_INPUT_MASS_MINIMUM_REFERENCE_TOTAL = 0.0 -# Independent publication pin for the active reviewed reference source. The data -# shard cannot import the build shard, so keep this in lockstep with -# uk/gates.json reference_registry["efrs-post-calibration"].identity. -_UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" -_UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", - "vintage": "2024_25", -} -# Independent publication pin for the canonical -# {"reference": {"identity": ..., "totals": ...}} evidence emitted from the -# reviewed 131-column enhanced-FRS reference. The totals remain uncommitted -# under the UKDS EUL; keep this in lockstep with -# UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 in the build shard. -_UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" +# The reference kind the spec names (microcosm#1095): the current build of the +# cut's own line. The report records which build that was; the data shard +# cannot import the build shard, so keep this in lockstep with uk/gates.json. +_UK_INPUT_MASS_ACTIVE_REFERENCE = "current_build" +_UK_INPUT_MASS_REFERENCE_IDENTITY_FIELDS = frozenset( + {"filename", "revision", "sha256", "vintage"} +) +_UK_CUT_REFERENCE_RELEASES = frozenset( + {"microcosm-uk-2024-25-national", "microcosm-uk-2024-25-dense"} ) _UK_TERMINAL_GATE_DETAIL_FIELDS = { "uk_release_input_coverage": frozenset( @@ -575,13 +567,13 @@ def _is_hierarchy_diagnostics_version(value: object) -> bool: # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "0d148ddccb8f126e4d64cfce94e820cec8d2adaccfc57fb87bb5f5ef2fc09106" + "ae9257b96da095b1c950f57727c0abb2c392e0cae8be23498f70eb3b6fb2a3f9" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "d3b13d6ce2a87aa0d9b87066cd46c3aa03b6c031dbb4c9d5ec44dc33e273bf63" + "75d36c4b3001aa0c2d902da239e4f1a99f5693ad2b2083f74b82093ed0700126" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "a7c86e189ecaea1061f7e280f0c63d5a6e13449e4a1f3a1be618eae8909a12e0" + "23bcfbf72d5e1f28b4d127cb79e8513b0c9b8de2c9bf2c61aab139a7e4475906" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -819,13 +811,10 @@ def _is_hierarchy_diagnostics_version(value: object) -> bool: "uk_cgt_projection_entrants", } ) -# The input-mass binding's evidence payload wraps the reviewed reference -# digest as {"reference_evidence_sha256": ...} before the executor's -# canonical hash; this pins the wrapped digest so the entry's evidence line -# still binds the enhanced-FRS incumbent totals. -_UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "24bb96f93de3bda45425b797a2f5069df7477bf3a56616080cc04376beab56e5" -) +# The input-mass binding's evidence payload digests the reference build's +# own totals (microcosm#1095), so it differs from cut to cut and is not pinned: +# the entry's details name the build, and its evidence line binds that build's +# totals. # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest # is a vintage pin that moves with every reviewed register edit (the same @@ -1147,10 +1136,10 @@ def _uk_line_cut_tag_re(release_id: str) -> re.Pattern[str] | None: }, "release_cut": { "gates_manifest_sha256": ( - "21c3d7bc17d2e80dccc87d9d063f96dd51f6f93e9631cf429bdc7fbc7be16fc3" + "c829049dcb1f52d3903985ecd71d01e21f09b1193bf11d47550f0fa47dc301e1" ), "policy_sha256": ( - "49bd5dbb2a744ad9c61bd67c67ccfaf34a3d25a0fb321acd4a98b060d123b8bb" + "76511c61eaf7a026165a2056c75dc79e525b0d12d017fbd902e5ecda2d95cd51" ), }, } @@ -2404,6 +2393,30 @@ def _check_uk_terminal_gate_links( ) +def _uk_input_mass_reference_named(identity: object, label: object) -> bool: + """Whether an input-mass report names the UK build it compared with.""" + + if not isinstance(identity, Mapping) or set(identity) != set( + _UK_INPUT_MASS_REFERENCE_IDENTITY_FIELDS + ): + return False + if not isinstance(label, str) or label.count("@") != 1: + return False + release, build = label.split("@", 1) + sha256 = identity.get("sha256") + return ( + release in _UK_CUT_REFERENCE_RELEASES + and bool(build) + and identity.get("revision") == build + and isinstance(sha256, str) + and _SHA256_RE.fullmatch(sha256) is not None + and all( + isinstance(identity.get(field), str) and identity.get(field) + for field in ("filename", "vintage") + ) + ) + + def _uk_terminal_observable_matches(observed: object, expected: object) -> bool: """Compare JSON observables without letting booleans masquerade as counts.""" @@ -2639,12 +2652,14 @@ def _check_uk_terminal_gate_observables( f"{_UK_TERMINAL_GATE_REPORT_FILE} input_mass_parity.details." "reference_scope_note must be a non-empty string." ) - identity = input_mass.get("reference_identity") - if identity != _UK_INPUT_MASS_REFERENCE_IDENTITY: + if not _uk_input_mass_reference_named( + input_mass.get("reference_identity"), input_mass.get("reference_label") + ): failures.append( - f"{_UK_TERMINAL_GATE_REPORT_FILE} input_mass_parity.details." - "reference_identity must match the active reviewed " - f"efrs-post-calibration reference {_UK_INPUT_MASS_REFERENCE_IDENTITY}." + f"{_UK_TERMINAL_GATE_REPORT_FILE} input_mass_parity.details must " + "name the reference build it compared with: reference_label " + "@ of a UK line, and reference_identity with that " + "build's file name and sha256." ) if not _uk_terminal_observable_matches( input_mass.get("relative_tolerance"), @@ -2952,15 +2967,6 @@ def _check_uk_terminal_gate_report( f"{_UK_TERMINAL_GATE_REPORT_FILE} attestation.evidence_sha256 " "must exactly match build_manifest.json terminal_gate_evidence." ) - if ( - "input_mass_parity" in build_evidence - and evidence.get("input_mass_parity") - != _UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 - ): - failures.append( - f"{_UK_TERMINAL_GATE_REPORT_FILE} input_mass_parity evidence " - "digest must bind the reviewed enhanced-FRS incumbent totals." - ) diagnostic_weights: Mapping | None = None if calibration_diagnostics is not None: uk = calibration_diagnostics.get("uk_diagnostics") @@ -3332,15 +3338,6 @@ def _check_uk_gate_battery_report( value=digest, failures=failures, ) - if ( - "uk_input_mass_parity" in evidence - and evidence.get("uk_input_mass_parity") - != _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 - ): - failures.append( - f"{file} uk_input_mass_parity evidence digest must bind the " - "reviewed enhanced-FRS incumbent totals." - ) if ( "uk_degenerate_release_surface" in evidence and evidence.get("uk_degenerate_release_surface") diff --git a/packages/microcosm-data/tests/engine_free/shared/test_contract.py b/packages/microcosm-data/tests/engine_free/shared/test_contract.py index 80327cb422..7087eda89e 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_contract.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_contract.py @@ -89,47 +89,26 @@ "release_parity": ("export_surface", "target_surface", "target_fit"), "input_mass_parity": ("input_mass_parity",), } +UK_INPUT_MASS_REFERENCE_BUILD = "uk-frs-calibration-attempt-20261107T020434Z-eeb6093e" +UK_INPUT_MASS_REFERENCE_LABEL = ( + f"microcosm-uk-2024-25-national@{UK_INPUT_MASS_REFERENCE_BUILD}" +) UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2024_25.h5", - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", + "filename": "microcosm_uk_2024_25.h5", + "revision": UK_INPUT_MASS_REFERENCE_BUILD, + "sha256": "9" * 64, "vintage": "2024_25", } -UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" -) -UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" +UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = "8" * 64 +UK_INPUT_MASS_ACTIVE_REFERENCE = "current_build" UK_INPUT_MASS_REFERENCE_SCOPE_NOTE = ( - "Channel-blind post-calibration enhanced-FRS production incumbent, pinned " - "to the 2024-25 line; structurally lacks the SPI clone channel, so " - "SPI-channel-exclusive columns are comparable only through per-reference " - "reviewed exclusions." + "The current build of the cut's own release line, as releases.json named " + "it when the cut fetched its inputs (microcosm#1095): totals at each " + "build's own solved weights, on the export surface." ) -# The committed per-reference register (microcosm#1063 c9): the policy payloads -# sealed into the input-mass evidence digest. -UK_INPUT_MASS_REVIEWED_EXCLUSIONS: dict[str, dict[str, str]] = { - "dfe_education_spending": { - "reason": "The candidate carries DfE education spending through the build (GBP 98.8bn against the incumbent's GBP 52.8m, +187,061%): the enhanced-FRS reference carries almost none of this input, so the comparison measures the reference's omission, not the candidate's. The input stays on the release surface and in the coverage manifest.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1063 c9 (certifier rehearsal R5 on the 2026-09-30 build, 2026-10-01; ruling 2026-10-02 in session)", - "approved_on": "2026-10-02", - "expires_on": "2027-04-02", - }, - "property_wealth": { - "reason": "The candidate no longer ships household.property_wealth: policyengine-uk derives it as the sum of main_residence_value, other_residential_property_value and non_residential_property_value, and a persisted copy overrides that sum and has no uprating index (microcosm#1106, the uk-data#543 defect). The enhanced-FRS reference stores its own WAS total, so the column is present in the reference and absent from the candidate, which this gate otherwise fails whatever the tolerance. The gate compares the release surface (the export frame, reviewed exclusions dropped) on both paths, the national seam and the release cut; the three components stay compared.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1106 (#1145): part B measured on the A9 candidate at 2025 and on the final-head arm c7 of 2026-10-11; ruling 2026-10-11 in session", - "approved_on": "2026-10-11", - "expires_on": "2027-04-11", - }, - "tax_free_savings_income": { - "reason": "The candidate no longer ships person.tax_free_savings_income: policyengine-uk derives it as the sum of individual_savings_account_interest_income, which the candidate stores apart from non-ISA interest under the additive ISA income contract, and a persisted copy overrides that sum and is never uprated (microcosm#1360). The enhanced-FRS reference stores it in the legacy gross layout, so the column is present in the reference and absent from the candidate, which this gate otherwise fails whatever the tolerance. Non-ISA savings_interest_income stays compared; it falls below the reference's gross total by the ISA part.", - "approved_by": "juaristi22", - "adjudication": "microcosm#1360 folded into microcosm#1418's PR B (ruling 2026-10-28 in session)", - "approved_on": "2026-10-28", - "expires_on": "2027-04-28", - }, -} +# The committed per-reference register is empty since the enhanced-FRS +# reference retired (microcosm#1095). +UK_INPUT_MASS_REVIEWED_EXCLUSIONS: dict[str, dict[str, str]] = {} GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" DATASET_SHA = "d" * 64 CALIBRATION_SHA = "a" * 64 @@ -227,20 +206,20 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "0d148ddccb8f126e4d64cfce94e820cec8d2adaccfc57fb87bb5f5ef2fc09106" + "ae9257b96da095b1c950f57727c0abb2c392e0cae8be23498f70eb3b6fb2a3f9" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "d3b13d6ce2a87aa0d9b87066cd46c3aa03b6c031dbb4c9d5ec44dc33e273bf63" + "75d36c4b3001aa0c2d902da239e4f1a99f5693ad2b2083f74b82093ed0700126" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "a7c86e189ecaea1061f7e280f0c63d5a6e13449e4a1f3a1be618eae8909a12e0" + "23bcfbf72d5e1f28b4d127cb79e8513b0c9b8de2c9bf2c61aab139a7e4475906" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "f246388de42654dd68dbbb40c42282001b5b0a8bf48976162bcb1199085cb1d3" ) -UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "24bb96f93de3bda45425b797a2f5069df7477bf3a56616080cc04376beab56e5" -) +# The input-mass evidence digest covers the reference build's own totals, so +# it is not pinned (microcosm#1095); a fixture value stands in. +UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = "7" * 64 #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). #: microcosm#1240: the two report-only high-value homes gates. UK_DIAGNOSTIC_ENTRY_IDS = frozenset( @@ -1182,7 +1161,7 @@ def _terminal_gate_details(name: str) -> dict: return { "candidate_name": "uk_release_candidate", "reference_name": UK_INPUT_MASS_REFERENCE_IDENTITY["filename"], - "relative_tolerance": 4.521811483823806, + "relative_tolerance": 5.9984285545209834, "minimum_reference_total": 0.0, "columns_checked": 1, "columns_below_reference_floor": 0, @@ -1198,6 +1177,8 @@ def _terminal_gate_details(name: str) -> dict: "reference": UK_INPUT_MASS_ACTIVE_REFERENCE, "reference_scope_note": UK_INPUT_MASS_REFERENCE_SCOPE_NOTE, "reference_identity": dict(UK_INPUT_MASS_REFERENCE_IDENTITY), + "reference_label": UK_INPUT_MASS_REFERENCE_LABEL, + "withheld_thin_columns": [], } if name == "take_up_signal": # Mirrors uk_take_up_signal_gate's per-column detail block. @@ -3248,37 +3229,33 @@ def test_exact_k_uk_terminal_rejects_unreviewed_input_mass_identity( validate_release_dir(directory) failures = "\n".join(excinfo.value.failures) - assert "reference_identity must match the active reviewed" in failures + assert "must name the reference build it compared with" in failures -def test_exact_k_uk_terminal_rejects_substituted_input_mass_totals( +def test_exact_k_uk_terminal_requires_the_reference_label( tmp_path: Path, ) -> None: + """The report must say which build it compared with (microcosm#1095).""" + directory = _write_uk_release_dir( tmp_path, UK_EXACT_K_RELEASE_ID, tier="frs", ) - payload, _evidence = _terminal_gate_payload( + payload, evidence = _terminal_gate_payload( release_id=UK_EXACT_K_RELEASE_ID, calibration_diagnostics_sha256=_sha256( directory / "calibration_diagnostics.json" ), evidence_stages=("input_mass_parity",), ) - substituted = _canonical_sha256( - { - "reference": { - "identity": dict(UK_INPUT_MASS_REFERENCE_IDENTITY), - "totals": {"employment_income": 1.0}, - } - } + payload["gates"]["input_mass_parity"]["details"]["reference_label"] = ( + "microcosm-uk-2024-25-national@another-build" ) - payload["attestation"]["evidence_sha256"]["input_mass_parity"] = substituted _refresh_terminal_gate_attestation(payload) build_path = directory / "build_manifest.json" build = json.loads(build_path.read_text()) - build["terminal_gate_evidence"] = dict(payload["attestation"]["evidence_sha256"]) + build["terminal_gate_evidence"] = evidence build_path.write_text(json.dumps(build)) _write_terminal_and_refresh_manifest_hashes(directory, payload) @@ -3286,7 +3263,7 @@ def test_exact_k_uk_terminal_rejects_substituted_input_mass_totals( validate_release_dir(directory) failures = "\n".join(excinfo.value.failures) - assert "must bind the reviewed enhanced-FRS incumbent totals" in failures + assert "must name the reference build it compared with" in failures def test_exact_k_uk_terminal_rejects_evidence_gate_without_its_stage( @@ -5006,18 +4983,7 @@ def test_uk_gate_battery_pins_are_in_lockstep_with_the_contract() -> None: contract_module._UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 == UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 ) - assert ( - contract_module._UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 - == UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 - ) - assert UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 == _canonical_sha256( - { - "reference": UK_INPUT_MASS_ACTIVE_REFERENCE, - "reference_evidence_sha256": UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, - "exclusions_policy": "committed", - "reviewed_exclusions": UK_INPUT_MASS_REVIEWED_EXCLUSIONS, - } - ) + assert not hasattr(contract_module, "_UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256") def test_exact_k_uk_gate_battery_report_validates_end_to_end(tmp_path: Path) -> None: @@ -5310,9 +5276,12 @@ def test_exact_k_uk_gate_battery_rejects_a_recorded_signing_error( assert "attestation must contain exactly" in _battery_failures(directory) -def test_exact_k_uk_gate_battery_rejects_an_unpinned_input_mass_evidence( +def test_exact_k_uk_gate_battery_accepts_any_reference_builds_evidence( tmp_path: Path, ) -> None: + """The input-mass evidence digest covers the reference build's totals, + which change with every cut, so no digest is pinned (microcosm#1095).""" + directory, payload = _write_battery_release(tmp_path) payload["evidence_sha256"]["uk_input_mass_parity"] = "b" * 64 _rewrite_battery_report(directory, payload) @@ -5322,7 +5291,7 @@ def test_exact_k_uk_gate_battery_rejects_an_unpinned_input_mass_evidence( build_path.write_text(json.dumps(build)) _refresh_terminal_manifest_hashes(directory) - assert "bind the reviewed enhanced-FRS" in _battery_failures(directory) + validate_release_dir(directory) def test_exact_k_uk_gate_battery_rejects_mixed_manifest_evidence_vocabulary( diff --git a/test_support/microcosm_build/uk_cut_reference.py b/test_support/microcosm_build/uk_cut_reference.py index 3832e114ee..c45eb56039 100644 --- a/test_support/microcosm_build/uk_cut_reference.py +++ b/test_support/microcosm_build/uk_cut_reference.py @@ -185,3 +185,36 @@ def __call__( return str(self._files[filename]) except KeyError as exc: raise missing_on_hub(repo_id, filename, revision) from exc + + +def stub_reference_evidence(reference, h5_path, output_dir): + """Write a reference's evidence without reading its dataset as a frame. + + The stand-in datasets here are a few bytes, not H5 files; the input-mass + file carries the identity the done-checks read and one synthetic column. + """ + + from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_INPUT_MASS_NAME, + ) + + destination = Path(output_dir) / UK_CUT_REFERENCE_INPUT_MASS_NAME + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps( + { + "schema_version": 2, + "kind": "uk_input_mass_reference", + "label": reference.label, + "identity": { + "filename": reference.filename, + "revision": reference.build, + "sha256": reference.sha256, + "vintage": "2024_25", + }, + "totals": {"employment_income": 1.0}, + "carriers": {"employment_income": 3}, + } + ) + ) + return {"input_mass_reference": destination} diff --git a/tools/build_uk_efrs_parity_reference.py b/tools/build_uk_efrs_parity_reference.py index 74e0076c43..d4f56447f1 100644 --- a/tools/build_uk_efrs_parity_reference.py +++ b/tools/build_uk_efrs_parity_reference.py @@ -116,8 +116,9 @@ def _parse_args() -> argparse.Namespace: metavar="PATH", help=( "Write weighted per-column totals of the artifact's effective " - "input surface, in the schema the input_mass_parity terminal gate " - "consumes (#609), and exit. This does NOT regenerate the committed " + "input surface, in the schema-1 shape the input_mass_parity terminal " + "gate consumed (#609) until microcosm#1095 moved it to the current " + "build of the cut's own line, and exit. This does NOT regenerate the committed " "coverage reference: the destination must be outside the " "repository, because the totals are an operational gate input " "derived from licensed microdata." diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 7c5042316a..27898626c4 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -352,7 +352,7 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: local_ledger_unsupported=local_ledger_unsupported, parity_evidence=parity_evidence, fit_weight_records=rehydrate_uk_fit_weight_records(sidecar), - input_mass_reference=load_uk_input_mass_reference(args.input_mass_reference), + input_mass_reference=_input_mass_reference(args), exclusions_evaluated_on=evaluated_on, spine_frame=spine_frame, ) @@ -393,6 +393,25 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: } +def _input_mass_reference(args: argparse.Namespace): + """The cut's input-mass reference, refused unless it is the cut's reference. + + The score and the input-mass gate compare with one build (microcosm#1095): + the reference record names it, and the input-mass file must too. + """ + + reference = json.loads(args.reference_record.read_text()) + loaded = load_uk_input_mass_reference(args.input_mass_reference) + label = f"{reference.get('release')}@{reference.get('build')}" + if loaded.label != label or loaded.sha256 != reference.get("sha256"): + raise SystemExit( + f"error: the input-mass reference names {loaded.label} " + f"({loaded.sha256}), not the cut's reference {label}; write it from " + "the cut's reference (tools/run_uk_cut.py's reference step)." + ) + return loaded + + def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-h5", required=True, type=Path) diff --git a/tools/resolve_uk_reference.py b/tools/resolve_uk_reference.py index 2f114dea09..3d8bbb6b6a 100644 --- a/tools/resolve_uk_reference.py +++ b/tools/resolve_uk_reference.py @@ -13,9 +13,11 @@ ``--check`` reads the release record, the build's manifest and its certification evidence, and downloads no dataset. ``--output-dir`` downloads the dataset, verifies its SHA-256, copies it there and writes -``reference.json`` beside it; pass both paths to the build's -``--reference-*`` options. It refuses a directory that already holds a -reference, so a run keeps the one it started with. +``reference.json`` beside it, with ``input_mass_reference.json`` (the +build's input-mass totals and carriers, licensed-data aggregates that stay +with the run). A dense build takes ``--input-mass-reference`` from it. It +refuses a directory that already holds a reference, so a run keeps the one +it started with. Reading ``policyengine/populace-uk-private`` needs ``HF_TOKEN``. """ @@ -33,6 +35,7 @@ UK_CUT_REFERENCE_RELEASES, check_uk_cut_reference, fetch_uk_cut_reference, + write_uk_cut_reference_evidence, ) @@ -47,26 +50,47 @@ def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: return parser.parse_args(argv) -def fetch_reference(release: str, output_dir: Path, *, hub_download=None) -> dict: - """Download the current build of ``release`` into ``output_dir``.""" +def fetch_reference( + release: str, + output_dir: Path, + *, + hub_download=None, + write_evidence=write_uk_cut_reference_evidence, +) -> dict: + """Download the current build of ``release`` into ``output_dir``. + + Beside the dataset and its record go the evidence the gates read from it: + its input-mass totals and carriers (``input_mass_reference.json``), which + a dense build takes with ``--input-mass-reference``. + """ reference = fetch_uk_cut_reference(release, output_dir, hub_download=hub_download) + written = write_evidence(reference, output_dir / reference.filename, output_dir) return { "reference": reference.label, "sha256": reference.sha256, "dataset": str(output_dir / reference.filename), "record": str(output_dir / UK_CUT_REFERENCE_RECORD_NAME), + **{name: str(path) for name, path in written.items()}, } -def main(argv: Sequence[str] | None = None, *, hub_download=None) -> int: +def main( + argv: Sequence[str] | None = None, + *, + hub_download=None, + write_evidence=write_uk_cut_reference_evidence, +) -> int: args = _parse_args(argv) try: if args.check: summary = check_uk_cut_reference(args.release, hub_download=hub_download) else: summary = fetch_reference( - args.release, args.output_dir.expanduser(), hub_download=hub_download + args.release, + args.output_dir.expanduser(), + hub_download=hub_download, + write_evidence=write_evidence, ) except ValueError as exc: print(f"refused: {exc}", file=sys.stderr) diff --git a/tools/run_uk_cut.py b/tools/run_uk_cut.py index 7787640f4a..4105f016bd 100644 --- a/tools/run_uk_cut.py +++ b/tools/run_uk_cut.py @@ -63,12 +63,15 @@ from typing import Any, Protocol from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_INPUT_MASS_NAME, UK_CUT_REFERENCE_RECORD_NAME, UKCutReference, check_uk_cut_reference, fetch_uk_cut_reference, read_uk_cut_reference, + uk_cut_reference_evidence_done, verify_uk_cut_reference_file, + write_uk_cut_reference_evidence, ) REPO_ROOT = Path(__file__).resolve().parents[1] @@ -611,6 +614,10 @@ def reference_record(self) -> Path: def reference_h5(self, reference: UKCutReference) -> Path: return self.reference_dir / reference.filename + @property + def reference_input_mass(self) -> Path: + return self.reference_dir / UK_CUT_REFERENCE_INPUT_MASS_NAME + @property def score_receipt(self) -> Path: return self.run_dir / "score_vs_reference.json" @@ -741,7 +748,6 @@ def certify_argv( logbook_prev_row_digest: str | None = None, ) -> list[str]: chronicle = {pin.name: pin.sha256 for pin in sources["chronicle_uk"].files} - reference = sources["input_mass_reference"].files[0] argv = [ python, str(REPO_ROOT / "tools/certify_uk_release_cut.py"), @@ -757,8 +763,7 @@ def certify_argv( "--ledger-facts", str(paths.inputs / "chronicle_uk"), "--ledger-facts-sha256", chronicle["consumer_facts.jsonl"], "--ledger-manifest-sha256", chronicle["manifest.json"], - "--input-mass-reference", - str(paths.input_file("input_mass_reference", reference.name)), + "--input-mass-reference", str(paths.reference_input_mass), "--score-receipt", str(paths.score_receipt), "--reference-record", str(paths.reference_record), "--release-id", RELEASE_ID, @@ -890,6 +895,9 @@ def reference_done(paths: CutPaths) -> tuple[bool, str]: verify_uk_cut_reference_file(reference, paths.reference_h5(reference)) except ValueError as error: return False, str(error) + evidence, detail = uk_cut_reference_evidence_done(reference, paths.reference_dir) + if not evidence: + return False, detail return True, f"compared with {reference.label}" @@ -1513,9 +1521,18 @@ def run_spine(runner: Runner, *, staging_local_only: bool, retry: bool = False) def run_reference( - runner: Runner, *, re_resolve: bool = False, hub_download=None + runner: Runner, + *, + re_resolve: bool = False, + hub_download=None, + write_evidence=write_uk_cut_reference_evidence, ) -> UKCutReference: - """Resolve the current national build once per cut, and keep its record.""" + """Resolve the current national build once per cut, and keep its record. + + Beside the dataset and its record go the evidence the gates read from it + (its input-mass totals and carriers). A reference whose evidence is + missing gets it written again; nothing re-resolves. + """ paths = runner.paths if re_resolve and paths.reference_dir.exists(): @@ -1528,6 +1545,18 @@ def run_reference( if done: runner.log(f"[reference] done: {detail}") return read_uk_cut_reference(paths.reference_record) + if paths.reference_record.exists(): + reference = read_uk_cut_reference(paths.reference_record) + try: + verify_uk_cut_reference_file(reference, paths.reference_h5(reference)) + except ValueError: + pass + else: + write_evidence( + reference, paths.reference_h5(reference), paths.reference_dir + ) + runner.log(f"[reference] wrote the evidence for {reference.label}") + return reference if paths.reference_dir.exists(): raise CutError( f"{paths.reference_dir} holds an unusable reference ({detail}); pass " @@ -1539,6 +1568,7 @@ def run_reference( reference = fetch_uk_cut_reference( RELEASE_ID, paths.reference_dir, hub_download=hub_download ) + write_evidence(reference, paths.reference_h5(reference), paths.reference_dir) except ValueError as error: runner.record("reference", status="failed", error=str(error)[:600]) raise CutError(f"the cut has no reference to compare with: {error}") from error diff --git a/tools/uk_cut_inputs.json b/tools/uk_cut_inputs.json index 83ffcbcb47..d311206274 100644 --- a/tools/uk_cut_inputs.json +++ b/tools/uk_cut_inputs.json @@ -1,6 +1,6 @@ { "version": 1, - "about": "Where tools/run_uk_cut.py fetches each input of a Microcosm UK national cut. This file says only where; every expected sha256 and size is read from the committed pins (uk/source_stages.json, uk/frs_release.json, uk/chronicle_feed.json, weighted_integrity.py), so a file is accepted only if it is byte-identical to what the build itself will check. A *_pin field reads its value from those pins instead: path_in_repo_pin chronicle_feed puts the Chronicle artifact under chronicle-uk/, so it follows every re-pin. revision null reads the repository's main branch, and every file is still verified against its pin (docs/uk-cut-anyone.md).", + "about": "Where tools/run_uk_cut.py fetches each input of a Microcosm UK national cut. This file says only where; every expected sha256 and size is read from the committed pins (uk/source_stages.json, uk/frs_release.json, uk/chronicle_feed.json), so a file is accepted only if it is byte-identical to what the build itself will check. A *_pin field reads its value from those pins instead: path_in_repo_pin chronicle_feed puts the Chronicle artifact under chronicle-uk/, so it follows every re-pin. revision null reads the repository's main branch, and every file is still verified against its pin (docs/uk-cut-anyone.md).", "sources": [ { "key": "frs_2024_25", @@ -138,25 +138,6 @@ "manifest.json", "consumer_facts.jsonl" ] - }, - { - "key": "input_mass_reference", - "kind": "hf_files", - "repo_id": "policyengine/microcosm-uk-build-inputs", - "repo_type": "dataset", - "revision": "95e0cb20fb37811c3f9009854d882a7415194926", - "path_in_repo": "input-mass-reference/fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", - "directory": "input_mass_reference", - "members": [ - "uk_input_mass_reference.json" - ] - } - ], - "file_pins": { - "uk_input_mass_reference.json": { - "sha256": "b3490f5bbc46581f2b5f81560a249b8983e37f80c67f00988cd3ed960a7f81c4", - "canonical_evidence_sha256": "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", - "note": "Byte pin of the hosted file; the build's own loader checks the canonical evidence digest. Regenerated at microcosm 0eac9bbbd (policyengine-uk 2.89.0); see the receipt beside it." } - } + ] } From 86630fc920675fdd071db458191bca54b1d74fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:31:20 +0000 Subject: [PATCH 05/18] Compare UK export surface and input coverage with the current build Part of #1095. The export-surface and release-input-coverage gates compared every UK cut with the frozen enhanced-FRS reference. A cut now compares with the current build of its own line, the build its reference step records. - The reference step writes reference_surface.json beside the reference: the columns the build exports and the loader inputs it carries, names and flags only. An input is carried by the coverage gate's own rule (uk_effective_input_signal: non-default signal on the reviewed share of effective mass), flagged thin when one or two source households carry it. - Export surface: the reference side is that surface. The declared extras shrink from 250 to the 26 the current national build does not export (the dense line's geography and clone columns); an entry the reference carries is reported as redundant, not failed. - Input coverage: the required inputs are the ones the reference carries, less the reviewed exclusions. The manifest moves to schema 4 and keeps the exclusions, the effective-mass policy and the stage-family contracts; the preflight stops checking the frozen enhanced-FRS surface and ledger. The gate refuses a reference measured at another floor, and an empty required set. - The certifier takes --reference-surface and refuses one that names another build than the cut's reference; run_uk_cut passes it. - Dense builds take --reference-surface as a graph source, beside --input-mass-reference, and refuse the two if they name different builds; without it the two gates are evidence_absent. The national role refuses it. - Battery digests re-pinned in contract.py and its test copies. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/uk/gates.json | 235 +--------- .../uk/release_input_coverage_manifest.json | 442 +----------------- .../build/uk_runtime/battery_bindings.py | 14 +- .../build/uk_runtime/cut_reference.py | 60 ++- .../build/uk_runtime/full_build_cli.py | 27 +- .../microcosm/build/uk_runtime/full_gates.py | 10 +- .../build/uk_runtime/graph_terminal.py | 33 +- .../build/uk_runtime/parity_reference.py | 9 +- .../build/uk_runtime/reference_surface.py | 200 ++++++++ .../build/uk_runtime/release_certification.py | 17 +- .../uk_runtime/release_input_coverage.py | 285 ++++++----- .../microcosm/build/uk_runtime/rowwise_cli.py | 4 + .../build/uk_runtime/terminal_gates.py | 359 ++------------ .../uk/test_uk_release_input_coverage.py | 240 +++++----- .../uk/test_uk_weight_share_formula_inputs.py | 45 +- .../uk/test_uk_battery_bindings.py | 64 ++- .../uk/test_uk_certify_release_cut_command.py | 53 +++ .../engine_free/uk/test_uk_frs_employment.py | 9 +- .../uk/test_uk_frs_relationships.py | 14 +- .../engine_free/uk/test_uk_full_gates.py | 5 - .../engine_free/uk/test_uk_graph_terminal.py | 22 +- .../uk/test_uk_hmrc_uprating_engine_pins.py | 1 + .../uk/test_uk_reference_surface.py | 158 +++++++ .../uk/test_uk_release_certification.py | 11 +- .../uk/test_uk_release_input_coverage.py | 138 +++++- ...test_uk_release_input_coverage_manifest.py | 41 +- .../uk/test_uk_rowwise_national_role.py | 17 + .../engine_free/uk/test_uk_terminal_gates.py | 58 ++- .../uk/test_uk_uc_deduction_attributes.py | 8 +- .../src/microcosm/data/contract.py | 10 +- .../tests/engine_free/shared/test_contract.py | 6 +- .../microcosm_build/uk_cut_reference.py | 40 +- .../uk_release_input_coverage.py | 6 + ...uild_uk_release_input_coverage_manifest.py | 96 ++-- tools/certify_uk_release_cut.py | 31 +- tools/resolve_uk_reference.py | 13 +- tools/run_uk_cut.py | 6 + 37 files changed, 1321 insertions(+), 1466 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/reference_surface.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_reference_surface.py diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 8d1f4d4240..c7cff46534 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -17,7 +17,7 @@ "parameters": { "check": "manifest_current" }, - "notes": "Frameless preflight mirror of the US cheap preflight: the committed coverage manifest must match the engine's current variable surface, aborting before source stages on graph or reference drift." + "notes": "Frameless preflight mirror of the US cheap preflight: the committed coverage manifest must agree with its sources (the effective-mass policy, the candidate and family tiers, the stage-family contracts and their source hashes, the restored inputs) and its declared columns must be live PolicyEngine-UK loader inputs, aborting before source stages. The required inputs come from the cut's reference build at gate time (microcosm#1095), so it no longer holds the manifest to the frozen enhanced-FRS surface." }, { "id": "uk_release_family_build_stages", @@ -1125,7 +1125,7 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": {}, - "notes": "The release input-coverage contract over the built frame: required columns present and populated per the committed manifest resource in this package." + "notes": "Every loader input the current build of the cut's line carries must be present on the release export and carry non-default signal on the reviewed share of effective population mass, less the manifest's reviewed exclusions; the reference carries an input by this gate's own rule, as the cut's reference step records in its reference surface (microcosm#1095). The stage families' build state is checked on the spine." }, { "id": "uk_degenerate_release_surface", @@ -1299,259 +1299,32 @@ "criticality": "release_blocking", "parameters": { "allowed_extra_columns": [ - "benunit.child_benefit_opts_out", - "benunit.frs_benunit_capital", - "benunit.has_mixed_age_couple_pension_credit_saving", - "benunit.liable_for_share_of_household_rent", - "benunit.pension_credit_reported_capital", - "benunit.uc_deduction_combination", - "benunit.uc_deduction_random_draw", - "benunit.uc_deduction_type_random_draw", - "benunit.uc_latent_deduction_rate", - "benunit.uc_reported_capital", - "benunit.would_claim_uc_at_legacy_closure", - "benunit.would_claim_uc_childcare", - "household.additional_residential_property_purchased", "household.atomic_area_basis", "household.atomic_area_code", "household.atomic_area_system", - "household.bus_fare_spending", - "household.bus_subsidy_spending", "household.cash_isa", - "household.cgt_support_copies", - "household.cgt_support_copy_index", - "household.high_value_support_copies", - "household.high_value_support_copy_index", - "household.household_is_high_value_support_copy", - "household.household_is_cgt_residential_clone", - "household.cgt_residential_clone_index", "household.household_clone_index", "household.constituency_code_oa", - "household.consumer_debt", "household.data_zone_code", "household.district_electoral_area_code", - "household.electricity_consumption", - "household.electricity_kwh", - "household.gas_consumption", - "household.gas_kwh", "household.geography_household_key", - "household.has_fuel_consumption", - "household.uses_energy_prepayment_meter", - "household.beer_litres", - "household.cider_litres", - "household.wine_litres", - "household.spirits_litres", - "household.other_fermented_litres", - "household.beer_draught_share", - "household.cider_draught_share", - "household.cigarettes_per_week", - "household.hand_rolling_tobacco_grams_per_week", - "household.cigars_grams_per_week", - "household.other_tobacco_grams_per_week", - "household.remote_gaming_net_losses", - "household.pool_betting_net_receipts", - "household.bingo_net_receipts", - "household.national_lottery_ticket_spending", - "household.land_based_casino_net_losses", - "household.machine_games_lower_rate_net_takings", - "household.machine_games_standard_rate_net_takings", - "household.machine_games_higher_rate_net_takings", - "household.land_based_fixed_odds_betting_net_losses", - "household.remote_uk_horseracing_betting_net_losses", - "household.remote_other_betting_net_losses", - "household.betting_exchange_commission", - "household.horse_and_dog_pool_betting_net_losses", - "household.household_is_capital_gains_clone", - "household.household_is_cgt_support_copy", - "household.household_is_spi_income_band_donor", - "household.household_is_spi_synthetic", - "household.household_lifetime_isa_balance", - "household.household_local_bus_trips", "household.in_rural_fuel_duty_relief_area", - "household.spi_income_band_donor_lower_bound", "household.intermediate_zone_code", "household.la_code_oa", "household.local_authority", "household.lsoa_code", - "household.mortgage_debt", "household.msoa_code", - "household.non_residential_property_purchased", - "household.num_vehicles", "household.oa_code", - "household.ons_household_type", "household.output_area_code", - "household.private_pension_wealth", - "household.property_purchased", - "household.rail_usage", "household.region_code_oa", "household.stocks_and_shares_isa", "household.super_data_zone_code", - "person.a_and_e_visits", - "person.aa_category", - "person.admitted_patient_visits", - "person.age_left_full_time_education", - "person.age_started_or_accepted_current_education_or_training", - "person.attends_private_school", - "person.attends_private_school_random_draw", - "person.capital_gains_asset_type", - "person.capital_gains_badr", - "person.capital_gains_residential_property", - "person.cgt_residential_probability", - "person.bus_in_london_trips", - "person.bus_pass_eligible", - "person.care_hours", - "person.charitable_investment_gifts", - "person.dla_m_category", - "person.dla_sc_category", - "person.employment_sector", - "person.contributory_benefits_annual_share", - "person.employment_income_annual_share", - "person.employment_income_other_spells", - "person.private_pension_income_annual_share", - "person.self_employment_income_annual_share", - "person.self_employment_income_other_spells", - "person.trading_allowance_claim_status", - "person.trading_allowance_full_relief", - "person.annuity_and_trust_income", - "person.esa_health_condition_proxy", - "person.esa_includes_support_component", - "person.esa_support_group_proxy", - "person.gift_aid", - "person.has_lifetime_isa", - "person.highest_education", - "person.is_before_universal_credit_qualifying_young_person_terminal_date", - "person.is_blind", - "person.is_claimant_or_partner", - "person.is_engaged_in_owned_company_trade", - "person.is_hbai_dependent_child", - "person.is_in_full_time_education", - "person.is_in_non_advanced_education", - "person.is_parent", - "person.is_uc_claimant", - "person.legacy_jobseeker_proxy", - "person.lifetime_isa_balance", - "person.lifetime_isa_payments", - "person.cash_isa_balance", - "person.stocks_and_shares_isa_balance", - "person.cash_isa_subscriptions", - "person.stocks_and_shares_isa_subscriptions", - "person.company_car_list_price", - "person.company_car_co2_emissions", - "person.company_car_fuel_type", - "person.company_car_electric_range", - "person.company_car_capital_contributions", - "person.company_car_private_fuel_provided", - "person.car_fuel_type", - "person.car_co2_emissions", - "person.car_engine_size", - "person.car_first_registration_date", - "person.car_is_ved_exempt", - "person.individual_savings_account_interest_income", - "person.local_bus_single_fare_share", - "person.local_bus_trips", - "person.local_bus_use_band", - "person.loss_relief", - "person.ni_class_4_losses_brought_forward", - "person.ons_family_index", - "person.ons_family_role", - "person.other_local_bus_trips", - "person.outpatient_visits", - "person.owned_company_carries_on_property_business", - "person.owned_company_carries_on_trade", - "person.owned_company_income_share", - "person.owned_company_is_main_employment", - "person.pension_contributions_via_salary_sacrifice", - "person.pip_dl_points", - "person.pip_m_points", - "person.property_finance_costs", - "person.property_rental_income", - "person.receives_benefits_in_own_right", - "person.rent_paid_as_boarder", - "person.rent_paid_as_lodger", - "person.relationship_to_head", - "person.reports_outstanding_student_loan", - "person.salary_sacrifice_asked", - "person.salary_sacrifice_reported", - "person.sic_industry_division", - "person.soc2020_major_group", - "person.soc2020_sub_major_group", - "person.stands_as_sole_owner_or_partner_of_company", - "person.statutory_paternity_pay", - "person.student_loan_balance", - "person.student_loan_plan", - "person.tax_free_childcare_spend_routed_share", - "person.is_apprentice", - "person.trading_loss", - "person.uc_is_in_startup_period", - "person.uc_is_in_gainful_self_employment", - "person.uc_limited_capability_for_WRA", - "person.uc_limited_capability_for_work", - "person.is_looked_after_by_local_authority", - "person.child_limit_non_parental_caring_exception", - "person.would_claim_carers_allowance", - "person.would_claim_marriage_allowance", - "person.would_claim_scp", - "person.max_free_entitlement_hours_used", - "person.person_is_spi_income_band_carrier", - "person.person_is_spi_income_recipient", - "benunit.benunit_source_id", - "benunit.benunit_support_channel", - "benunit.benunit_support_clone_index", - "benunit.dependent_children", - "household.council_tax_rebate", - "household.council_tax_reported", - "household.council_tax_single_adult_raw", - "household.household_source_id", - "household.household_support_channel", - "household.household_support_clone_index", - "household.num_bedrooms", - "household.source_household_id", - "household.source_household_key", - "household.source_year", - "household.subrent", - "person.disabled_students_allowance_eligible_expenses", - "person.free_school_breakfasts", - "person.hmrc_spi_assessable_income", - "person.hmrc_spi_employed_income", - "person.hmrc_spi_employment_benefits", - "person.hmrc_spi_employment_expenses", - "person.hmrc_spi_incapacity_benefit_income", - "person.hmrc_spi_miscellaneous_employment_income", - "person.hmrc_spi_other_income", - "person.hmrc_spi_other_social_security_income", - "person.hmrc_spi_pay", - "person.hmrc_spi_state_pension_income", - "person.hmrc_spi_taxable_termination_pay", - "person.hmrc_spi_total_earned_income", - "person.hmrc_spi_total_investment_income", - "person.hmrc_spi_unemployment_benefit_income", - "person.is_in_approved_training", - "person.ossben_identifiable_subset", - "person.other_investment_income", - "person.person_source_id", - "person.person_support_channel", - "person.person_support_clone_index", - "person.srp_regular_code5", "benunit.benunit_clone_index", "person.person_clone_index", - "benunit.benunit_build_id", - "household.household_build_id", - "person.person_build_id", - "person.months_since_last_birthday", "household.itl1_code", "household.itl2_code", "household.itl3_code", - "household.ward_code", - "person.war_widows_pension_reported", - "person.widowed_parents_allowance_reported", - "person.spousal_maintenance_income", - "person.pension_contributions_via_bonus_sacrifice", - "household.liquid_fuel_consumption", - "household.solid_fuel_consumption", - "household.directly_held_shares", - "household.unit_and_investment_trusts", - "household.other_residential_property_secured_debt", - "household.non_residential_property_secured_debt" + "household.ward_code" ], "reviewed_exclusions": { "household.property_wealth": "policyengine-uk derives property_wealth from main_residence_value, other_residential_property_value and non_residential_property_value; a persisted copy overrides that sum and has no uprating index, so the candidate must drop it (microcosm#1106, the uk-data#543 defect).", @@ -1560,7 +1333,7 @@ "person.tax_free_savings_income": "policyengine-uk derives tax_free_savings_income as the sum of the ISA interest it adds (individual_savings_account_interest_income); a persisted copy overrides that sum and is never uprated, so the candidate stores ISA interest apart from non-ISA interest under the additive ISA income contract and drops it (microcosm#1360)." } }, - "notes": "Reviewed candidate-only fields from the June UK prototype (source provenance or genuine additional model inputs, not incumbent-surface losses), with household.uses_energy_prepayment_meter added for microcosm#1110 (the LCFS prepayment-meter flag policyengine-uk reads from 2.123.3 for the Energy Price Guarantee prepayment discount; an older engine ignores it, and the incumbent does not carry it until uk-data#539), the LCFS excise step's 24 household inputs added for microcosm#1359 (alcohol litres and the beer and cider draught shares, tobacco a week and the gambling duty bases; policyengine-uk reads the alcohol and tobacco ones from 2.145.0 and the gambling ones from 2.151.0, an older engine ignores them, and the incumbent carries none) plus the reviewed reference-side exclusions of the formula-owned property_wealth total (microcosm#1106) and the formula-owned tax_free_savings_income total, whose ISA interest the candidate stores apart under the additive ISA income contract (microcosm#1360). The all-zero incapacity benefit layer left the exclusions (microcosm#1095, from #796): the pinned reference lists it among its zero-share inputs, not on its export surface, so its exclusion never matched; the release boundary still drops the column, and this gate would refuse it as an unlisted extra. The two hard-required columns that may never be waived stay a code-level guard, per the microcosm#327 rule that a parity comparison's registers are declared inputs. During the migration window the runtime merges the exclusion register with the legacy constant and the spec-pin tests hold the two equal; the register becomes solely load-bearing when the constant retires." + "notes": "The candidate's export surface against the current build of the cut's own line (microcosm#1095), whose exported columns the cut's reference step records: a column the reference exports must be exported or be a reviewed drop, and a column only the candidate exports must be declared here. The list holds only what the current national build does not export: the dense line's geography and clone columns, which its own reference carries, and columns new since the reference, each naming its PR. An entry the reference now carries is reported as redundant, never failed. The reviewed exclusions are columns a candidate must never export, since a persisted copy overrides the engine's formula. Until microcosm#1095 the reference was the enhanced FRS's 145 inputs, beside 250 declared extras." }, { "id": "uk_take_up_signal", diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 62fd1d755f..c628490af0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -8,309 +8,6 @@ "tier": "frs" }, "columns": { - "a_and_e_visits": { - "status": "required" - }, - "aa_category": { - "status": "required" - }, - "access_fund": { - "status": "required" - }, - "accommodation_type": { - "status": "required" - }, - "admitted_patient_visits": { - "status": "required" - }, - "adult_ema": { - "status": "required" - }, - "afcs_reported": { - "status": "required" - }, - "age": { - "status": "required" - }, - "age_started_or_accepted_current_education_or_training": { - "status": "required" - }, - "alcohol_and_tobacco_consumption": { - "status": "required" - }, - "attends_private_school_random_draw": { - "status": "required" - }, - "brma": { - "status": "required" - }, - "bsp_reported": { - "status": "required" - }, - "bus_subsidy_spending": { - "status": "required" - }, - "capital_gains": { - "status": "required" - }, - "carers_allowance_reported": { - "status": "required" - }, - "charitable_investment_gifts": { - "status": "required" - }, - "child_benefit_opts_out": { - "status": "required" - }, - "child_benefit_reported": { - "status": "required" - }, - "child_ema": { - "status": "required" - }, - "child_tax_credit_reported": { - "status": "required" - }, - "childcare_expenses": { - "status": "required" - }, - "clothing_and_footwear_consumption": { - "status": "required" - }, - "communication_consumption": { - "status": "required" - }, - "corporate_wealth": { - "status": "required" - }, - "council_tax": { - "status": "required" - }, - "council_tax_band": { - "status": "required" - }, - "council_tax_benefit_reported": { - "status": "required" - }, - "current_education": { - "status": "required" - }, - "dfe_education_spending": { - "status": "required" - }, - "diesel_spending": { - "status": "required" - }, - "dividend_income": { - "status": "required" - }, - "dla_m_category": { - "status": "required" - }, - "dla_sc_category": { - "status": "required" - }, - "domestic_energy_consumption": { - "status": "required" - }, - "domestic_rates": { - "status": "required" - }, - "education_consumption": { - "status": "required" - }, - "education_grants": { - "status": "required" - }, - "electricity_consumption": { - "status": "required" - }, - "employee_pension_contributions": { - "status": "required" - }, - "employer_pension_contributions": { - "status": "required" - }, - "employment_income": { - "status": "required" - }, - "employment_status": { - "status": "required" - }, - "esa_contrib_reported": { - "status": "required" - }, - "esa_income_reported": { - "status": "required" - }, - "external_child_payments": { - "status": "required" - }, - "food_and_non_alcoholic_beverages_consumption": { - "status": "required" - }, - "free_school_fruit_veg": { - "status": "required" - }, - "free_school_meals": { - "status": "required" - }, - "full_rate_vat_expenditure_rate": { - "status": "required" - }, - "gas_consumption": { - "status": "required" - }, - "gender": { - "status": "required" - }, - "gift_aid": { - "status": "required" - }, - "gross_financial_wealth": { - "status": "required" - }, - "health_consumption": { - "status": "required" - }, - "healthy_start_vouchers": { - "status": "required" - }, - "highest_education": { - "status": "required" - }, - "hours_worked": { - "status": "required" - }, - "household_furnishings_consumption": { - "status": "required" - }, - "household_owns_tv": { - "status": "required" - }, - "household_weight": { - "status": "required" - }, - "housing_benefit_reported": { - "status": "required" - }, - "housing_service_charges": { - "status": "required" - }, - "housing_water_and_electricity_consumption": { - "status": "required" - }, - "iidb_reported": { - "status": "required" - }, - "income_support_reported": { - "status": "required" - }, - "is_before_universal_credit_qualifying_young_person_terminal_date": { - "status": "required" - }, - "is_benunit_head": { - "status": "required" - }, - "is_disabled_for_benefits": { - "status": "required" - }, - "is_enhanced_disabled_for_benefits": { - "status": "required" - }, - "is_household_head": { - "status": "required" - }, - "is_in_non_advanced_education": { - "status": "required" - }, - "is_married": { - "status": "required" - }, - "is_parent": { - "status": "required" - }, - "is_severely_disabled_for_benefits": { - "status": "required" - }, - "jsa_contrib_reported": { - "status": "required" - }, - "jsa_income_reported": { - "status": "required" - }, - "lump_sum_income": { - "status": "required" - }, - "main_residence_value": { - "status": "required" - }, - "main_residential_property_purchased_is_first_home": { - "status": "required" - }, - "maintenance_expenses": { - "status": "required" - }, - "maintenance_income": { - "status": "required" - }, - "marital_status": { - "status": "required" - }, - "maximum_extended_childcare_hours_usage": { - "status": "required" - }, - "miscellaneous_consumption": { - "status": "required" - }, - "miscellaneous_income": { - "status": "required" - }, - "mortgage_capital_repayment": { - "status": "required" - }, - "mortgage_interest_repayment": { - "status": "required" - }, - "net_financial_wealth": { - "status": "required" - }, - "nhs_a_and_e_spending": { - "status": "required" - }, - "nhs_admitted_patient_spending": { - "status": "required" - }, - "nhs_outpatient_spending": { - "status": "required" - }, - "non_residential_property_value": { - "status": "required" - }, - "num_vehicles": { - "status": "required" - }, - "other_residential_property_value": { - "status": "required" - }, - "outpatient_visits": { - "status": "required" - }, - "owned_land": { - "status": "required" - }, - "pension_contributions_via_salary_sacrifice": { - "status": "required" - }, - "pension_credit_reported": { - "status": "required" - }, - "personal_pension_contributions": { - "status": "required" - }, - "petrol_spending": { - "status": "required" - }, "pip_dl_category": { "reason": "formula-owned in policyengine-uk and derived from persisted components; a persisted copy overrides the formula and is not uprated", "status": "reviewed_exclusion", @@ -321,144 +18,22 @@ "status": "reviewed_exclusion", "tracking_note": "microcosm#1095 (P1 of the November event-data plan): policyengine-uk 2.174.1 derives pip_m_category from pip_m_points (8 points for the standard rate, 12 for the enhanced) where no category is supplied for the year; the pip_points stage stores the points within the stored award, and the release and calibration boundaries drop the category, which the spine keeps (María's signature of 2026-11-09)." }, - "private_pension_income": { - "status": "required" - }, - "private_transfer_income": { - "status": "required" - }, - "property_income": { - "status": "required" - }, - "property_purchased": { - "status": "required" - }, "property_wealth": { "reason": "formula-owned in policyengine-uk and derived from persisted components; a persisted copy overrides the formula and is not uprated", "status": "reviewed_exclusion", "tracking_note": "microcosm#1106 (the uk-data#543 defect): the release and calibration boundaries drop the WAS total, and policyengine-uk sums main_residence_value, other_residential_property_value and non_residential_property_value, which stay required." }, - "rail_subsidy_spending": { - "status": "required" - }, - "rail_usage": { - "status": "required" - }, - "receives_benefits_in_own_right": { - "status": "required" - }, - "recreation_consumption": { - "status": "required" - }, - "region": { - "status": "required" - }, - "rent": { - "status": "required" - }, - "restaurants_and_hotels_consumption": { - "status": "required" - }, - "savings": { - "status": "required" - }, - "savings_interest_income": { - "status": "required" - }, - "sda_reported": { - "status": "required" - }, - "self_employment_income": { - "status": "required" - }, - "ssmg_reported": { - "status": "required" - }, - "state_pension_reported": { - "status": "required" - }, - "statutory_maternity_pay": { - "status": "required" - }, - "statutory_sick_pay": { - "status": "required" - }, - "structural_insurance_payments": { - "status": "required" - }, - "student_loan_balance": { - "status": "required" - }, - "student_loan_plan": { - "status": "required" - }, - "student_loan_repayments": { - "status": "required" - }, - "student_loans": { - "status": "required" - }, "tax_free_savings_income": { "reason": "formula-owned in policyengine-uk and derived from persisted components; a persisted copy overrides the formula and is not uprated", "status": "reviewed_exclusion", "tracking_note": "microcosm#1360: under the additive ISA income contract the producer stores ISA interest in individual_savings_account_interest_income, which policyengine-uk adds into tax_free_savings_income; savings_interest_income, now interest outside ISAs, stays required." - }, - "tenure_type": { - "status": "required" - }, - "transport_consumption": { - "status": "required" - }, - "universal_credit_reported": { - "status": "required" - }, - "water_and_sewerage_charges": { - "status": "required" - }, - "winter_fuel_allowance_reported": { - "status": "required" - }, - "working_tax_credit_reported": { - "status": "required" - }, - "would_claim_child_benefit": { - "status": "required" - }, - "would_claim_extended_childcare": { - "status": "required" - }, - "would_claim_marriage_allowance": { - "status": "required" - }, - "would_claim_pc": { - "status": "required" - }, - "would_claim_scp": { - "status": "required" - }, - "would_claim_targeted_childcare": { - "status": "required" - }, - "would_claim_tfc": { - "status": "required" - }, - "would_claim_uc": { - "status": "required" - }, - "would_claim_universal_childcare": { - "status": "required" - }, - "would_evade_tv_licence_fee": { - "status": "required" } }, "counts": { - "required": 141, - "reviewed_exclusion": 4, - "total": 145 + "reviewed_exclusion": 4 }, - "derivation": "Surface = efrs_parity_reference.json populated effective loader inputs. status='required' when the sha-pinned candidate evidence records non-default signal on at least the reviewed owning-entity effective-mass share OR the column is pinned in restored_required_columns after a source-family restoration; all remaining surface columns are reviewed_exclusion with reason 'not yet ported from enhanced FRS pipeline — pending review' and a UK_COVERAGE_PROGRESS.md tracking note, except the engine_derived_exclusions, which are reviewed_exclusion with their own reason because the engine computes them from persisted inputs. The final release gate applies the same effective-mass floor, and distributional restorations must pass it on their required source channel.", - "description": "Declared full-coverage contract for a UK release: every populated effective loader input in the pinned enhanced FRS must be persisted with non-default signal, or carry a reviewed exclusion with the campaign reason and tracking note.", + "derivation": "Columns = the reviewed exclusions of efrs_parity_known_gaps.json: its known gaps, with reason 'not yet ported from enhanced FRS pipeline — pending review' and a UK_COVERAGE_PROGRESS.md tracking note, and its engine_derived_exclusions, with their own reason because the engine computes them from persisted inputs. Every other input the cut's reference build carries is required. The gate applies one effective-mass floor to the reference and the candidate, and distributional restorations must pass it on their required source channel.", + "description": "Release input-coverage contract for a UK cut: every loader input the current build of the cut's line carries must be persisted with non-default signal on the reviewed share of effective population mass, unless it carries a reviewed exclusion here with its reason and tracking note (microcosm#1095).", "effective_mass_coverage": { "minimum_nondefault_mass_share": 1e-06, "rationale": "One part per million rejects zero-weight support and numerical dust while remaining about 100 times below the rarest populated record share in the pinned enhanced-FRS reference.", @@ -1238,13 +813,8 @@ } }, "reference": { - "derived_from": "efrs_parity_reference.json", - "filename": "enhanced_frs_2024_25.h5", - "period": "2024", - "populated_input_columns": 145, - "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", - "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", - "vintage": "2024_25" + "kind": "current_build", + "required_inputs": "The loader inputs the current build of the cut's line carries, measured by the coverage gate's own rule; the cut's reference step records them in reference_surface.json. Until microcosm#1095 they were the populated inputs of the frozen enhanced-FRS reference." }, "restoration_evidence": { "derived_from": "efrs_parity_known_gaps.json", @@ -1253,5 +823,5 @@ "gift_aid" ] }, - "schema_version": 3 + "schema_version": 4 } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index ab7cba76f3..a7153ff636 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -258,9 +258,13 @@ def _evaluate_release_input_coverage( build_state = _uk_gate_surface(spine_frame) else: build_state = spine_frame + # The required inputs are the ones the cut's reference build carries + # (microcosm#1095); the artifact selector marks the gate evidence_absent + # when no reference surface arrived. return uk_release_input_coverage_gate( _uk_gate_surface(uk_release_export_frame(context.frame)), engine, + reference_surface=context.artifacts["reference_surface"], manifest=manifest, build_state_frame=build_state, ) @@ -277,7 +281,7 @@ def _coverage_required_artifacts(parameters: Mapping[str, Any]) -> frozenset[str # the calibrated frame. if parameters.get("check") == "manifest_current": return frozenset({"coverage_engine"}) - return frozenset({"coverage_engine", "spine_frame"}) + return frozenset({"coverage_engine", "spine_frame", "reference_surface"}) def _evaluate_source_coverage( @@ -1489,10 +1493,14 @@ def _evaluate_weights_audit( def _evaluate_export_surface( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: + # The reference side is the export surface of the cut's reference build, + # the current build of the line (microcosm#1095). parity = context.artifacts["parity_evidence"] + reference = context.artifacts["reference_surface"] return uk_export_surface_gate( parity.candidate_columns, - parity.reference_columns, + reference.export_columns, + reference_name=reference.label, **dict(parameters), ) @@ -2386,7 +2394,7 @@ def _ledger_compile_parity_required_artifacts( name="export_surface", evaluator=_evaluate_export_surface, parameter_keys=frozenset({"allowed_extra_columns", "reviewed_exclusions"}), - artifact_keys=frozenset({"parity_evidence"}), + artifact_keys=frozenset({"parity_evidence", "reference_surface"}), needs_frame=False, ), "target_surface": UKGateBinding( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py index d60438e064..0a76f27160 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cut_reference.py @@ -43,6 +43,7 @@ "UKCutReference", "UK_CUT_REFERENCE_INPUT_MASS_NAME", "UK_CUT_REFERENCE_RECORD_NAME", + "UK_CUT_REFERENCE_SURFACE_NAME", "check_uk_cut_reference", "fetch_uk_cut_reference", "read_uk_cut_reference", @@ -60,6 +61,9 @@ #: The reference's input-mass totals and carriers, written beside it: the #: input-mass gate's reference (licensed-data aggregates, never committed). UK_CUT_REFERENCE_INPUT_MASS_NAME = "input_mass_reference.json" +#: The reference's column surface (names and flags): the export-surface and +#: input-coverage gates' reference (:mod:`.reference_surface`). +UK_CUT_REFERENCE_SURFACE_NAME = "reference_surface.json" #: Each line's registry variant and the build evidence its fetch reads. UK_CUT_REFERENCE_RELEASES: Mapping[str, tuple[str, str]] = { @@ -477,11 +481,15 @@ def write_uk_cut_reference_evidence( """Write the evidence the gates read from the reference, beside it. The input-mass totals and carriers of the build's export frame at its own - solved weights (:func:`~.weighted_integrity.uk_input_mass_reference_payload`). - The file holds licensed-data aggregates: it stays with the cut, as the - reference's dataset does. + solved weights (:func:`~.weighted_integrity.uk_input_mass_reference_payload`), + which hold licensed-data aggregates and stay with the cut as the + reference's dataset does, and the build's column surface, names and + flags only (:func:`~.reference_surface.uk_reference_surface_payload`). """ + from microcosm.build.uk_runtime.reference_surface import ( + uk_reference_surface_payload, + ) from microcosm.build.uk_runtime.weighted_integrity import ( uk_input_mass_reference_payload, ) @@ -496,11 +504,20 @@ def write_uk_cut_reference_evidence( revision=reference.build, sha256=reference.sha256, ) - destination = directory / UK_CUT_REFERENCE_INPUT_MASS_NAME - partial = destination.with_name(destination.name + ".partial") - partial.write_text(json.dumps(payload, sort_keys=True, allow_nan=False)) - partial.replace(destination) - return {"input_mass_reference": destination} + surface = uk_reference_surface_payload( + source, label=reference.label, sha256=reference.sha256 + ) + written: dict[str, Path] = {} + for key, name, content in ( + ("input_mass_reference", UK_CUT_REFERENCE_INPUT_MASS_NAME, payload), + ("reference_surface", UK_CUT_REFERENCE_SURFACE_NAME, surface), + ): + destination = directory / name + partial = destination.with_name(destination.name + ".partial") + partial.write_text(json.dumps(content, sort_keys=True, allow_nan=False)) + partial.replace(destination) + written[key] = destination + return written def uk_cut_reference_evidence_done( @@ -508,16 +525,19 @@ def uk_cut_reference_evidence_done( ) -> tuple[bool, str]: """Whether the reference's evidence is beside it and names this build.""" - path = Path(output_dir) / UK_CUT_REFERENCE_INPUT_MASS_NAME - try: - payload = json.loads(path.read_text()) - except (OSError, ValueError): - return False, f"no {UK_CUT_REFERENCE_INPUT_MASS_NAME} beside the reference" - identity = payload.get("identity") if isinstance(payload, Mapping) else None - if ( - not isinstance(identity, Mapping) - or payload.get("label") != reference.label - or identity.get("sha256") != reference.sha256 - ): - return False, f"{UK_CUT_REFERENCE_INPUT_MASS_NAME} names another build" + for name in (UK_CUT_REFERENCE_INPUT_MASS_NAME, UK_CUT_REFERENCE_SURFACE_NAME): + try: + payload = json.loads((Path(output_dir) / name).read_text()) + except (OSError, ValueError): + return False, f"no {name} beside the reference" + if not isinstance(payload, Mapping): + return False, f"{name} is not a JSON object" + identity = payload.get("identity") + sha256 = ( + identity.get("sha256") + if isinstance(identity, Mapping) + else payload.get("sha256") + ) + if payload.get("label") != reference.label or sha256 != reference.sha256: + return False, f"{name} names another build" return True, f"evidence for {reference.label}" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 361b711041..49e2221829 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -603,6 +603,16 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--measure-exclusions", type=Path) parser.add_argument("--register-json", type=Path) parser.add_argument("--input-mass-reference", type=Path) + parser.add_argument( + "--reference-surface", + type=Path, + help=( + "Dense role: the export surface and carried inputs of the dense " + "line's current build (reference_surface.json from " + "tools/resolve_uk_reference.py), which the export-surface and " + "input-coverage gates compare with (microcosm#1095)." + ), + ) parser.add_argument( "--native-scorecard", type=Path, @@ -1462,14 +1472,18 @@ def prepare_full_build( if path is not None: sources[name] = path optional.append(name) - if args.input_mass_reference is not None: - sources["uk_input_mass_reference"] = args.input_mass_reference + for name, path in ( + ("uk_input_mass_reference", args.input_mass_reference), + # The reference build's surface (microcosm#1095): the export-surface + # and input-coverage gates read it, and are evidence_absent without. + ("uk_reference_surface", getattr(args, "reference_surface", None)), + ): + if path is None: + continue + sources[name] = path spine = replace( spine, - sources=( - *spine.sources, - SourceRef("uk_input_mass_reference", "raw-bytes-v1"), - ), + sources=(*spine.sources, SourceRef(name, "raw-bytes-v1")), ) checkpoint_identity = ( None @@ -1809,6 +1823,7 @@ def _argument_sources(args: argparse.Namespace) -> dict[str, Path]: ("measure_exclusions", args.measure_exclusions), ("register_json", args.register_json), ("input_mass_reference", args.input_mass_reference), + ("reference_surface", getattr(args, "reference_surface", None)), ("native_scorecard", args.native_scorecard), ("matched_size_scorecard", args.matched_size_scorecard), ("resume_size_checkpoint", args.resume_size_checkpoint), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py index f9c9d7b183..6d3d201960 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py @@ -25,7 +25,6 @@ from microcosm.build.uk_runtime.cgt_projection import UK_CGT_PROJECTION_ARTIFACT_KEY from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_artifacts from microcosm.build.uk_runtime.national_frame import uk_release_export_frame -from microcosm.build.uk_runtime.parity_reference import load_efrs_parity_reference from microcosm.build.uk_runtime.support_floor import uk_support_floor_names from microcosm.build.uk_runtime.terminal_gates import uk_export_candidate_columns from microcosm.calibrate import TargetRegistry @@ -254,7 +253,6 @@ def build_full_gate_context( for spec in reference_registry.specs if (spec.name, spec.period) in selected } - reference = load_efrs_parity_reference() manifest = uk_full_gate_manifest(selection_receipt) admin_totals, admin_receipt = uk_aggregate_admin_totals(frame, manifest) artifacts = dict(supporting_evidence) @@ -281,14 +279,14 @@ def build_full_gate_context( "resolved_reference_count": len(problem.targets), "matrix_target_count": len(problem.names), }, + # The export-surface and input-coverage gates read the reference + # build's surface (``reference_surface``, the dense line's + # current build, from --reference-surface) beside this; without + # it they are evidence_absent (microcosm#1095). "parity_evidence": SimpleNamespace( candidate_columns=uk_export_candidate_columns( uk_release_export_frame(frame) ), - reference_columns={ - f"{entity}.{name}" - for name, entity in reference.input_entities.items() - }, candidate_targets=set(national_errors), reference_targets=national_reference, target_relative_errors=national_errors, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 5e55da28b9..0cff774ce2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -651,6 +651,25 @@ def decode_full_gate_report(payload: bytes | Mapping): return report, enforcement +def _require_one_reference(supporting: Mapping[str, Any]) -> None: + """Refuse an input-mass reference and a surface of two different builds. + + The dense line compares with one build, its current one (microcosm#1095): + both files come from ``tools/resolve_uk_reference.py`` for that build. + """ + + mass = supporting.get("input_mass_reference") + surface = supporting.get("reference_surface") + if mass is None or surface is None: + return + if (mass.label, mass.sha256) != (surface.label, surface.sha256): + raise ValueError( + f"The input-mass reference names {mass.label} ({mass.sha256}) but the " + f"reference surface names {surface.label} ({surface.sha256}); write " + "both from one reference with tools/resolve_uk_reference.py." + ) + + def _spine_gate_evidence(context: KernelContext): from ..stage_evidence import decode_stage_evidence @@ -846,6 +865,7 @@ def run(self, context: KernelContext) -> KernelResult: ) from .geography_ladder import load_uk_oa_ladder from .local_rowwise import uk_ladder_area_support_summary + from .reference_surface import load_uk_reference_surface from .release_certification import rehydrate_uk_fit_weight_records from .weighted_integrity import load_uk_input_mass_reference @@ -938,6 +958,11 @@ def run(self, context: KernelContext) -> KernelResult: supporting["input_mass_reference"] = load_uk_input_mass_reference( context.sources["uk_input_mass_reference"] ) + if "uk_reference_surface" in context.sources: + supporting["reference_surface"] = load_uk_reference_surface( + context.sources["uk_reference_surface"] + ) + _require_one_reference(supporting) household = frame.table("household").copy() household["household_weight"] = frame.weights_for("household").values summaries = uk_ladder_area_support_summary( @@ -1209,10 +1234,10 @@ def append_uk_full_gate_nodes( else node for node in graph.nodes ) - sources = ("uk_ladder",) + ( - ("uk_input_mass_reference",) - if any(source.name == "uk_input_mass_reference" for source in graph.sources) - else () + sources = ("uk_ladder",) + tuple( + name + for name in ("uk_input_mass_reference", "uk_reference_surface") + if any(source.name == name for source in graph.sources) ) # The spine checkpoint's build state, when the bound checkpoint publishes # it: the input-coverage gate reads the stages' importance weights and diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/parity_reference.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/parity_reference.py index a9114ff953..0006755aab 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/parity_reference.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/parity_reference.py @@ -1,9 +1,12 @@ -"""Pinned enhanced-FRS input-surface reference for UK coverage parity. +"""Pinned enhanced-FRS input-surface reference, kept as a historical record. The licensed H5 never ships in Microcosm. ``efrs_parity_reference.json`` is the frozen, sha-addressed extraction of its populated PolicyEngine-UK input leaves. -Release coverage manifests and gates consume this small checked-in fact instead -of resolving a moving or licensed artifact at build time. +Until microcosm#1095 the release coverage manifest and the export-surface and +input-coverage gates compared every UK cut with it; a cut now compares with the +current build of its own line (:mod:`.reference_surface`). The coverage +manifest generator still checks its exclusions ledger, +``efrs_parity_known_gaps.json``, against these frozen facts it was measured on. """ from __future__ import annotations diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/reference_surface.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/reference_surface.py new file mode 100644 index 0000000000..cac1860b6f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/reference_surface.py @@ -0,0 +1,200 @@ +"""The column surface of a cut's reference build (microcosm#1095). + +The export-surface and release-input-coverage gates compared every UK cut +with the frozen enhanced-FRS parity reference: the columns it exported and +the inputs it populated. A cut now compares with the current build of its +own line, so the reference step writes that build's surface beside it: + +- ``export_columns``: the ``entity.column`` names its release frame exports, + as :func:`~.terminal_gates.uk_export_candidate_columns` reads a candidate; +- the loader inputs it carries, by the input-coverage gate's own rule + (:func:`~.release_input_coverage.uk_effective_input_signal`: non-default + signal against the live engine default on at least + ``minimum_nondefault_mass_share`` of effective population mass), split by + how many distinct source households carry them on rows of positive weight: + ``populated_inputs`` at least ``UK_DISCLOSURE_FLOOR``, ``thin_inputs`` one + or two (a count below the floor reads "<3"). + +Names and flags only: the file holds no figure beyond the policy floor it was +measured at. Its identity names the build (``label``, ``sha256``), and a gate +refuses a surface of another build than the cut's reference. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pandas as pd + +__all__ = [ + "UK_REFERENCE_SURFACE_KIND", + "UK_REFERENCE_SURFACE_SCHEMA_VERSION", + "UKReferenceSurface", + "load_uk_reference_surface", + "uk_reference_surface_payload", +] + +UK_REFERENCE_SURFACE_KIND = "uk_reference_surface" +UK_REFERENCE_SURFACE_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class UKReferenceSurface: + """What the reference build exports and populates, by name.""" + + label: str + sha256: str + export_columns: frozenset[str] + populated_inputs: frozenset[str] + thin_inputs: frozenset[str] + minimum_nondefault_mass_share: float + + @property + def carried_inputs(self) -> frozenset[str]: + """Every input the reference carries with signal, thin ones included.""" + + return self.populated_inputs | self.thin_inputs + + +def uk_reference_surface_payload( + h5_path: str | Path, + *, + label: str, + sha256: str, + engine: Any | None = None, +) -> dict[str, Any]: + """The surface payload of the build at ``h5_path``, names and flags only. + + ``engine`` is the coverage engine whose loader inputs and defaults the + gate will judge the candidate by (the live PolicyEngine-UK adapter when + omitted); the floor is the committed coverage manifest's. + """ + + from microcosm.build.uk_runtime.national_frame import ( + _uk_gate_surface, + load_uk_national_frame, + uk_release_export_frame, + ) + from microcosm.build.uk_runtime.receipt_counts import ( + UK_DISCLOSURE_FLOOR, + distinct_count, + source_household_ids, + ) + from microcosm.build.uk_runtime.release_input_coverage import ( + PolicyEngineUKCoverageEngine, + load_uk_release_input_coverage_manifest, + uk_effective_input_signal, + ) + from microcosm.build.uk_runtime.terminal_gates import uk_export_candidate_columns + from microcosm.build.uk_runtime.weight_year import uk_recorded_solve_year_frame + + if engine is None: + engine = PolicyEngineUKCoverageEngine() + floor = load_uk_release_input_coverage_manifest().effective_mass_coverage + floor = floor.minimum_nondefault_mass_share + frame, _provenance = load_uk_national_frame(h5_path) + export = uk_release_export_frame(uk_recorded_solve_year_frame(frame)) + surface = _uk_gate_surface(export) + schema = export.schema + household = surface.household + household_sources = pd.Series( + source_household_ids(household, stage="reference surface"), + index=household[schema.id_column("household")].to_numpy(), + ) + person = surface.person + person_sources = household_sources.reindex( + person[schema.membership_column("household")].to_numpy() + ).to_numpy() + first_member = ( + pd.DataFrame( + { + "benunit": person[schema.membership_column("benunit")].to_numpy(), + "source": person_sources, + } + ) + .drop_duplicates("benunit") + .set_index("benunit")["source"] + ) + sources_by_entity = { + "household": household_sources.to_numpy(), + "person": person_sources, + "benunit": first_member.reindex( + surface.benunit[schema.id_column("benunit")].to_numpy() + ).to_numpy(), + } + populated: set[str] = set() + thin: set[str] = set() + carried = uk_effective_input_signal( + surface, engine, minimum_nondefault_mass_share=floor + ) + for name, (entity, signal) in carried.items(): + # ``signal`` already excludes rows of zero weight. + carriers = distinct_count(sources_by_entity[entity], signal) + (populated if carriers >= UK_DISCLOSURE_FLOOR else thin).add(str(name)) + return { + "schema_version": UK_REFERENCE_SURFACE_SCHEMA_VERSION, + "kind": UK_REFERENCE_SURFACE_KIND, + "label": label, + "sha256": sha256, + "minimum_nondefault_mass_share": floor, + "export_columns": sorted(uk_export_candidate_columns(export)), + "populated_inputs": sorted(populated), + "thin_inputs": sorted(thin), + } + + +def load_uk_reference_surface(source: str | Path) -> UKReferenceSurface: + """Load a reference surface a cut wrote, refusing any other shape.""" + + path = Path(source) + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError(f"{path}: a reference surface must be a JSON object.") + if ( + payload.get("schema_version") != UK_REFERENCE_SURFACE_SCHEMA_VERSION + or payload.get("kind") != UK_REFERENCE_SURFACE_KIND + ): + raise ValueError( + f"{path}: not a schema-{UK_REFERENCE_SURFACE_SCHEMA_VERSION} " + f"{UK_REFERENCE_SURFACE_KIND}." + ) + label = payload.get("label") + sha256 = payload.get("sha256") + if not isinstance(label, str) or label.count("@") != 1: + raise ValueError(f"{path}: a reference surface's label is @.") + if not isinstance(sha256, str) or len(sha256) != 64: + raise ValueError(f"{path}: a reference surface names its build's sha256.") + floor = payload.get("minimum_nondefault_mass_share") + if ( + isinstance(floor, bool) + or not isinstance(floor, int | float) + or not math.isfinite(float(floor)) + or not 0.0 < float(floor) <= 1.0 + ): + raise ValueError( + f"{path}: a reference surface names the effective-mass floor it " + "was measured at, in (0, 1]." + ) + lists = {} + for key in ("export_columns", "populated_inputs", "thin_inputs"): + value = payload.get(key) + if not isinstance(value, list) or not all( + isinstance(item, str) and item for item in value + ): + raise ValueError(f"{path}: {key} must be a list of column names.") + lists[key] = frozenset(value) + if lists["populated_inputs"] & lists["thin_inputs"]: + raise ValueError(f"{path}: a column cannot be both populated and thin.") + if not lists["export_columns"]: + raise ValueError(f"{path}: a reference surface exports no column.") + return UKReferenceSurface( + label=label, + sha256=sha256, + minimum_nondefault_mass_share=float(floor), + **lists, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 3181ec6ad1..6ebb537fee 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -232,12 +232,13 @@ def uk_release_parity_evidence( *, diagnostics_targets: Sequence[Mapping[str, Any]], reference_registry: Any, - parity_reference: Any, ) -> SimpleNamespace: """The parity-trio evidence over persisted inputs, sides never aliased. - Candidate columns come from the staged frame; reference columns from the - frozen parity instrument's declared input entities. Candidate targets + Candidate columns come from the staged frame; the export-surface gate + reads the reference's columns from the cut's reference surface itself + (the ``reference_surface`` artifact, microcosm#1095), where until then + they came from the frozen enhanced-FRS instrument. Candidate targets come from the solve's realized diagnostics rows; reference targets from the independently recompiled (and exclusion-pruned) register at ``name@period`` grain — the same two-source rule the retired June @@ -265,10 +266,6 @@ def uk_release_parity_evidence( target_relative_errors[name] = float(row["relative_error"]) return SimpleNamespace( candidate_columns=uk_export_candidate_columns(frame), - reference_columns={ - f"{entity}.{name}" - for name, entity in parity_reference.input_entities.items() - }, candidate_targets=set(target_relative_errors), reference_targets={ f"{spec.name}@{spec.period}" for spec in reference_registry.specs @@ -315,6 +312,7 @@ def run_uk_release_cut_battery( parity_evidence: Any, fit_weight_records: tuple[FitWeightRecord, ...] | None, input_mass_reference: Mapping[str, Any], + reference_surface: Any, exclusions_evaluated_on: date, gate_registry: Mapping[str, Any] | None = None, spine_frame: Any | None = None, @@ -324,7 +322,9 @@ def run_uk_release_cut_battery( ``spine_frame`` is the frame the spine build produced, the evidence the input-coverage gate's family build-state half reads (typed importance weights and each stage's mass receipt); the calibrated candidate is the - evidence for everything else. + evidence for everything else. ``reference_surface`` is the cut's + reference build's surface, whose carried inputs the coverage gate + requires (microcosm#1095). Always release-candidate strict: this battery exists to certify a cut, so an ``evidence_absent`` gap blocks rather than being tolerated, and a @@ -378,6 +378,7 @@ def run_uk_release_cut_battery( "parity_evidence": parity_evidence, "aggregate_admin": admin_totals, "input_mass_reference": input_mass_reference, + "reference_surface": reference_surface, "release_line": "national", } if fit_weight_records is not None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index 36ef09053d..1a387855e0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -1,12 +1,16 @@ -"""UK release input-column coverage: the enhanced-FRS surface as a hard gate. - -Every populated effective loader input extracted from the immutable enhanced -FRS reference is declared in ``release_input_coverage_manifest.json`` as either -``required`` or ``reviewed_exclusion``. Required columns must be present on the -final export tables and carry non-default signal on rows with enough effective -population mass to clear the reviewed manifest floor. A reviewed exclusion -that later carries effective signal is stale and fails, so the debt ledger can -only shrink deliberately. +"""UK release input-column coverage against the cut's reference build. + +Every loader input the current build of the cut's line carries must reach the +release export with non-default signal on rows carrying at least the reviewed +share of effective population mass, unless ``release_input_coverage_manifest.json`` +declares it a reviewed exclusion (microcosm#1095). The reference carries an +input by the same rule the gate applies to the candidate +(:func:`uk_effective_input_signal`), and the cut's reference step records +which inputs those are (:mod:`.reference_surface`); until microcosm#1095 the +required inputs were the populated inputs of the frozen enhanced-FRS +reference. The manifest may also declare an input required whatever the +reference carries. A reviewed exclusion that later carries effective signal +is stale and fails, so the debt ledger can only shrink deliberately. The UK loader calls ``set_input`` for every engine-known persisted H5 column, including formula-owned variables. Those persisted overrides are therefore @@ -29,11 +33,7 @@ import pandas as pd from microcosm.build.gates import GateResult, input_column_coverage_gate -from microcosm.build.uk_runtime.parity_reference import ( - load_efrs_engine_derived_exclusions, - load_efrs_parity_known_gaps, - load_efrs_parity_reference, -) +from microcosm.build.uk_runtime.reference_surface import UKReferenceSurface from microcosm.build.uk_runtime.release_identity import ( UK_RELEASE_TIER_FRS, validate_uk_release_tier, @@ -41,6 +41,7 @@ __all__ = [ "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS", + "UK_RELEASE_INPUT_COVERAGE_SCHEMA_VERSION", "UKEffectiveMassCoveragePolicy", "UK_FAMILY_MASS_CHANGE_SEMANTICS", "UK_LOADER_INPUT_ALIASES", @@ -51,12 +52,18 @@ "assert_uk_release_input_coverage_build_stages", "assert_uk_release_input_coverage_manifest_current", "load_uk_release_input_coverage_manifest", + "uk_effective_input_signal", "uk_release_input_coverage_gate", "uk_release_input_coverage_required_columns", "uk_release_input_coverage_reviewed_exclusions", ] UK_RELEASE_INPUT_COVERAGE_RESOURCE = "release_input_coverage_manifest.json" +#: Schema 4 (microcosm#1095): the required inputs come from the cut's +#: reference build at gate time, so the manifest declares the reviewed +#: exclusions, the effective-mass policy and the stage-family contracts. +UK_RELEASE_INPUT_COVERAGE_SCHEMA_VERSION = 4 +_REFERENCE_KIND = "current_build" UK_LOADER_INPUT_ALIASES: Mapping[str, str] = { "capital_gains": "capital_gains_before_response", "employee_pension_contributions": ("employee_pension_contributions_reported"), @@ -76,7 +83,6 @@ #: mass record conserves the total and declares factor one. UK_FAMILY_MASS_CHANGE_SEMANTICS = "mass_conserving" _UK_PACKAGE = "microcosm.build.uk" -_EFRS_PARITY_REFERENCE_RESOURCE = "efrs_parity_reference.json" DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE = 1e-6 @@ -433,10 +439,24 @@ def load_uk_release_input_coverage_manifest( ) -> UKReleaseInputCoverageManifest: """Load the non-vacuous UK manifest and validate exclusion metadata.""" payload = _resource_payload(resource) + schema_version = payload.get("schema_version") + if schema_version != UK_RELEASE_INPUT_COVERAGE_SCHEMA_VERSION: + raise ValueError( + f"{resource}: schema_version must be " + f"{UK_RELEASE_INPUT_COVERAGE_SCHEMA_VERSION}, got {schema_version!r}; " + "the required inputs now come from the cut's reference build " + "(microcosm#1095)." + ) raw_reference = payload.get("reference") raw_candidate = payload.get("candidate_evidence") if not isinstance(raw_reference, Mapping): raise ValueError(f"{resource}: 'reference' must be a JSON object.") + if raw_reference.get("kind") != _REFERENCE_KIND: + raise ValueError( + f"{resource}: reference.kind must be {_REFERENCE_KIND!r}; the " + "coverage contract compares a cut with the current build of its " + "own line." + ) if not isinstance(raw_candidate, Mapping): raise ValueError(f"{resource}: 'candidate_evidence' must be a JSON object.") try: @@ -461,11 +481,10 @@ def load_uk_release_input_coverage_manifest( ) raw_columns = payload.get("columns") - if not isinstance(raw_columns, Mapping) or not raw_columns: - raise ValueError( - f"{resource}: 'columns' must be a non-empty JSON object; a silently " - "empty manifest would make the coverage gate vacuous." - ) + if not isinstance(raw_columns, Mapping): + # Empty is allowed: the required inputs come from the reference, and + # the gate itself refuses a vacuous required set. + raise ValueError(f"{resource}: 'columns' must be a JSON object.") columns: list[UKReleaseInputColumn] = [] for name, entry in sorted(raw_columns.items()): if not isinstance(entry, Mapping): @@ -479,9 +498,6 @@ def load_uk_release_input_coverage_manifest( note=str(entry.get("note", "")), ) ) - schema_version = payload.get("schema_version", 1) - if not isinstance(schema_version, int): - raise ValueError(f"{resource}: 'schema_version' must be an integer.") return UKReleaseInputCoverageManifest( reference={str(key): str(value) for key, value in raw_reference.items()}, candidate_evidence={ @@ -499,6 +515,8 @@ def load_uk_release_input_coverage_manifest( def uk_release_input_coverage_required_columns() -> frozenset[str]: + """Inputs the committed manifest requires whatever the reference carries.""" + return load_uk_release_input_coverage_manifest().required_columns @@ -1058,15 +1076,70 @@ def _valid_mass_record(record: object) -> bool: return bool(np.isclose(old, new, rtol=1e-9, atol=0.0) and factor == 1.0) +def uk_effective_input_signal( + frame: Any, + engine: Any, + *, + minimum_nondefault_mass_share: float, +) -> dict[str, tuple[str, np.ndarray]]: + """Each loader input ``frame`` carries, by the coverage gate's own rule. + + An input is carried when its non-default signal (against the live engine + default) sits on at least ``minimum_nondefault_mass_share`` of its + entity's effective population mass: the floor the gate applies to a + candidate. The cut's reference step measures the reference build with + this, so a candidate is required to carry exactly what the reference + carries under the rule it is judged by (microcosm#1095). Returns each + carried input's entity and its row mask of signal on positive mass. + """ + + floor = float(minimum_nondefault_mass_share) + loadable = set(engine.variables()) + table_items = _entity_tables(frame) + tables = {entity: table for entity, table in table_items} + effective_weights = _effective_weights_by_entity(frame, tables) + present: dict[str, tuple[str, Any]] = {} + for entity, table in table_items: + for column in table.columns: + if column not in loadable: + continue + if column in present: + raise ValueError( + f"UK coverage column {column!r} occurs on both " + f"{present[column][0]!r} and {entity!r} tables." + ) + present[column] = (entity, table[column].to_numpy()) + defaults = engine.default_values(sorted(present)) + carried: dict[str, tuple[str, np.ndarray]] = {} + for name, (entity, values) in sorted(present.items()): + if name not in defaults: + continue + weights = np.asarray(effective_weights[entity], dtype=np.float64) + positive = weights > 0.0 + signal = _nondefault_signal_mask(values, defaults[name]) & positive + total_mass = float(weights[positive].sum()) + share = float(weights[signal].sum()) / total_mass if total_mass > 0 else 0.0 + if share >= floor: + carried[name] = (entity, signal) + return carried + + def uk_release_input_coverage_gate( frame: Any, engine: Any, *, + reference_surface: UKReferenceSurface, manifest: UKReleaseInputCoverageManifest | None = None, build_state_frame: Any | None = None, ) -> GateResult: """Enforce required signal on rows carrying reviewed effective mass. + The required inputs are the ones the reference build carries + (``reference_surface``, measured by :func:`uk_effective_input_signal`) + among the live engine's loader inputs, plus any the manifest declares + required, less the manifest's reviewed exclusions (microcosm#1095). The + reference must have been measured at the manifest's effective-mass floor. + The column-coverage and effective-mass halves read ``frame``, the release frame. The family build-state half (each stage family's typed ``importance`` weights and its mass receipt) is a spine-stage contract: @@ -1077,8 +1150,24 @@ def uk_release_input_coverage_gate( half on it fails every family on the kind alone. """ manifest = manifest or load_uk_release_input_coverage_manifest() - required = manifest.required_columns reviewed = manifest.reviewed_exclusions + floor = manifest.effective_mass_coverage.minimum_nondefault_mass_share + if reference_surface.minimum_nondefault_mass_share != floor: + raise ValueError( + f"The reference surface of {reference_surface.label} was measured at " + f"an effective-mass floor of " + f"{reference_surface.minimum_nondefault_mass_share!r}, not the " + f"manifest's {floor!r}; re-run the cut's reference step." + ) + loadable = set(engine.variables()) + from_reference = frozenset(reference_surface.carried_inputs & loadable) + required = (from_reference | manifest.required_columns) - set(reviewed) + if not required: + raise ValueError( + f"UK input coverage has no required input: the reference " + f"{reference_surface.label} carries no live loader input and the " + "manifest declares none, so the gate would be vacuous." + ) relevant = required | set(reviewed) expected_entities = _engine_variable_entities(engine, relevant) table_items = _entity_tables(frame) @@ -1120,7 +1209,6 @@ def uk_release_input_coverage_gate( for name, (_entity, values) in present_values.items() if not _nondefault_signal_mask(values, defaults[name]).any() } - floor = manifest.effective_mass_coverage.minimum_nondefault_mass_share insufficient_effective_mass = { name for name, details in diagnostics.items() @@ -1133,22 +1221,22 @@ def uk_release_input_coverage_gate( degenerate_columns=degenerate, reviewed_exclusions=reviewed, name="uk_release_input_coverage", - reference_label="enhanced-FRS", + reference_label="reference", ) failures = list(base.failures) for name, mismatch in wrong_entities.items(): failures.append( - f"{name}: enhanced-FRS input column is stored on " + f"{name}: reference input column is stored on " f"{mismatch['actual']!r}, but PolicyEngine-UK owns it on " f"{mismatch['expected']!r}; a same-named column on the wrong table " "does not satisfy loader coverage." ) for name in sorted((insufficient_effective_mass - raw_degenerate) & required): - prefix = f"{name}: required enhanced-FRS input column is present but every " + prefix = f"{name}: required reference input column is present but every " failures = [failure for failure in failures if not failure.startswith(prefix)] column_details = diagnostics[name] failures.append( - f"{name}: required enhanced-FRS input column carries non-default " + f"{name}: required reference input column carries non-default " f"signal on effective population mass share " f"{float(column_details['effective_signal_mass_share']):.12g}, below " f"the reviewed floor {floor:.12g}; zero-weight support or numerical " @@ -1170,6 +1258,12 @@ def uk_release_input_coverage_gate( failures.extend(family_build_failures) details = { **dict(base.details), + # Names only: which build the required inputs came from, and which + # of them it carries on fewer than three source households ("<3"). + "reference": reference_surface.label, + "required_from_reference": len(from_reference - set(reviewed)), + "required_by_manifest": sorted(manifest.required_columns), + "required_thin_in_reference": sorted(required & reference_surface.thin_inputs), "effective_mass_policy": { "weight_source": manifest.effective_mass_coverage.weight_source, "minimum_nondefault_mass_share": floor, @@ -1200,31 +1294,21 @@ def _coverage_engine() -> PolicyEngineUKCoverageEngine | None: return engine -def _efrs_reference_payload() -> Mapping[str, Any]: - return _resource_payload(_EFRS_PARITY_REFERENCE_RESOURCE) - - -def _efrs_populated_layers() -> frozenset[str]: - shares = _efrs_reference_payload().get("nonzero_shares") - if not isinstance(shares, Mapping) or not shares: - raise ValueError( - f"{_EFRS_PARITY_REFERENCE_RESOURCE}: 'nonzero_shares' must be a " - "non-empty JSON object." - ) - return frozenset(str(name) for name, share in shares.items() if float(share) > 0.0) - - -def _efrs_input_entities() -> dict[str, str]: - reference = load_efrs_parity_reference() - return dict(reference.input_entities) - - def assert_uk_release_input_coverage_manifest_current( *, engine: Any | None = None, manifest: UKReleaseInputCoverageManifest | None = None, ) -> None: - """Fail before a build if the UK contract has narrowed or graph-rotted.""" + """Fail before a build if the UK contract has drifted from its sources. + + The required inputs come from the cut's reference build at gate time + (microcosm#1095), so this preflight checks only what does not depend on + the reference: the effective-mass policy, the candidate and family + tiers, the restored inputs, the stage-family contracts and their source + hashes, and that every declared column is a live PolicyEngine-UK loader + input with an owning entity. Until then it also held the manifest to the + frozen enhanced-FRS surface and its known-gaps ledger. + """ manifest = manifest or load_uk_release_input_coverage_manifest() failures: list[str] = [] if manifest.effective_mass_coverage != UKEffectiveMassCoveragePolicy(): @@ -1246,31 +1330,11 @@ def assert_uk_release_input_coverage_manifest_current( f"UKDS-licensed candidate, got {candidate_tier!r}." ) declared = set(manifest.declared_columns) - surface = set(_efrs_populated_layers()) - reference_entities = _efrs_input_entities() - missing = sorted(surface - declared) - extra = sorted(declared - surface) - if missing: - failures.append( - "manifest is missing enhanced-FRS populated loader input(s) " - f"{missing}; regenerate with " - "tools/build_uk_release_input_coverage_manifest.py." - ) - if extra: - failures.append( - "manifest declares column(s) outside the enhanced-FRS populated " - f"loader-input surface: {extra}." - ) - for name in sorted(RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS): if name in manifest.reviewed_exclusions: failures.append( - f"{name}: restored enhanced-FRS reference input cannot return " - "to a reviewed exclusion." - ) - elif name not in manifest.required_columns: - failures.append( - f"{name}: restored enhanced-FRS reference input must remain required." + f"{name}: a restored reference input cannot return to a " + "reviewed exclusion." ) for family_name, family in sorted(manifest.family_coverage.items()): @@ -1321,11 +1385,11 @@ def assert_uk_release_input_coverage_manifest_current( for column, requirement in sorted(requirements.items()): if ( family_status == _REQUIRED_AT_BUILD_STATUS - and column not in manifest.required_columns + and column in manifest.reviewed_exclusions ): failures.append( - f"{family_name}: distributional requirement {column!r} must " - "be a required release input while the family is " + f"{family_name}: distributional requirement {column!r} " + "cannot be a reviewed exclusion while the family is " "required_at_build." ) if ( @@ -1346,63 +1410,6 @@ def assert_uk_release_input_coverage_manifest_current( "with the reviewed UK release policy." ) - known_gaps = {gap.variable: gap for gap in load_efrs_parity_known_gaps()} - # Engine-derived exclusions (microcosm#1106) are a separate section of the - # same ledger: reference inputs the engine computes from persisted inputs, - # which the release must therefore not persist. They are never parity debt. - engine_derived = { - gap.variable: gap for gap in load_efrs_engine_derived_exclusions() - } - both = sorted(set(known_gaps) & set(engine_derived)) - if both: - failures.append( - "efrs_parity_known_gaps.json names column(s) both as a parity known " - f"gap and as an engine-derived exclusion: {both}." - ) - known_gaps = {**known_gaps, **engine_derived} - gaps_outside_surface = sorted(set(known_gaps) - surface) - if gaps_outside_surface: - failures.append( - "efrs_parity_known_gaps.json declares column(s) outside the " - f"enhanced-FRS populated surface: {gaps_outside_surface}." - ) - for name in sorted(surface & declared): - column = manifest._by_name[name] - gap = known_gaps.get(name) - if gap is None: - if column.status != REQUIRED_STATUS: - failures.append( - f"{name}: efrs_parity_known_gaps.json has no gap, so the " - "manifest status must remain required." - ) - continue - if column.status != REVIEWED_EXCLUSION_STATUS: - failures.append( - f"{name}: committed known gap must be a reviewed_exclusion in " - "the manifest." - ) - continue - if column.reason != gap.reason: - failures.append( - f"{name}: manifest exclusion reason disagrees with " - "efrs_parity_known_gaps.json." - ) - if column.tracking_note != gap.tracking_note: - failures.append( - f"{name}: manifest exclusion tracking note disagrees with " - "efrs_parity_known_gaps.json." - ) - - raw_engine = _efrs_reference_payload().get("engine") - raw_aliases = ( - raw_engine.get("h5_input_aliases") if isinstance(raw_engine, Mapping) else None - ) - if raw_aliases != dict(UK_LOADER_INPUT_ALIASES): - failures.append( - "efrs_parity_reference.json h5_input_aliases disagree with the " - "live UK coverage adapter; regenerate or update both together." - ) - if engine is None: engine = _coverage_engine() if engine is not None: @@ -1418,23 +1425,9 @@ def assert_uk_release_input_coverage_manifest_current( ) if not non_inputs: try: - live_entities = _engine_variable_entities(engine, declared) + _engine_variable_entities(engine, declared) except ValueError as exc: failures.append(str(exc)) - else: - entity_drift = { - name: { - "reference": reference_entities[name], - "live": live_entities[name], - } - for name in sorted(surface & declared) - if reference_entities[name] != live_entities[name] - } - if entity_drift: - failures.append( - "enhanced-FRS input owning entities disagree with the " - f"live PolicyEngine-UK graph: {entity_drift}." - ) if failures: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 3d41a2664d..8ec6c77892 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -1200,6 +1200,10 @@ def refuse_dense_role_arguments( refused.append("--require-solve-reuse") if args.dataset_households is not None: refused.append("--dataset-households") + # The national cut's certify step reads its reference surface + # (microcosm#1095); the national build runs no export-surface gate. + if getattr(args, "reference_surface", None) is not None: + refused.append("--reference-surface") if args.selection_seed is not None: refused.append("--selection-seed") if getattr(args, "selection_initial_lambda", None) is not None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index b26c2fe22d..07fb5be649 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -73,12 +73,10 @@ ] UK_CANDIDATE_DATASET_NAME = "microcosm_uk_2024_25" -# The label names the pinned reference artifact exactly: the 2024-25 line's -# published enhanced_frs_2024_25.h5 (no separate "recalibrated" variant -# exists at this vintage; the June report strings keep their own label). -# After the swap, "reference" becomes the previous certified microcosm line -# once a second certified cut exists; that later increment flips this value. -UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25" +# The export-surface reference is the current build of the cut's own line +# (microcosm#1095), as the cut's reference step recorded it; the reports name +# the build. Until then it was the published enhanced_frs_2024_25.h5. +UK_REFERENCE_DATASET_NAME = "current_build" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 @@ -168,343 +166,41 @@ def __post_init__(self) -> None: ) -# Reviewed candidate-only fields from the June UK prototype. They are source -# provenance or genuine additional model inputs, not incumbent-surface losses. +# Reviewed columns a candidate may export beyond the reference's surface. +# Since microcosm#1095 the reference is the current build of the cut's own +# line, so this lists only what the current national build does not export: +# the dense line's geography and clone columns, which its own reference +# carries, and any column new since the reference, each naming its PR. An +# entry the reference now carries is reported as redundant, never failed; +# the enhanced-FRS era's 224 entries the current build carries left with +# it. UK_ALLOWED_EXTRA_EXPORT_COLUMNS: tuple[str, ...] = ( - "benunit.child_benefit_opts_out", - "benunit.frs_benunit_capital", - "benunit.has_mixed_age_couple_pension_credit_saving", - "benunit.liable_for_share_of_household_rent", - "benunit.pension_credit_reported_capital", - "benunit.uc_deduction_combination", - "benunit.uc_deduction_random_draw", - "benunit.uc_deduction_type_random_draw", - "benunit.uc_latent_deduction_rate", - "benunit.uc_reported_capital", - "benunit.would_claim_uc_at_legacy_closure", - "benunit.would_claim_uc_childcare", - "household.additional_residential_property_purchased", "household.atomic_area_basis", "household.atomic_area_code", "household.atomic_area_system", - "household.bus_fare_spending", - "household.bus_subsidy_spending", "household.cash_isa", - # #1045: the CGT support split's family copy count. - "household.cgt_support_copies", - "household.cgt_support_copy_index", - # microcosm#1240: the high-value homes support split's family copy count, - # index and flag, the CGT split's precedent. The stage's coverage-manifest - # outputs; the first dense release build failed this gate on them - # (microcosm#1301). - "household.high_value_support_copies", - "household.high_value_support_copy_index", - "household.household_is_high_value_support_copy", - # #1063: the residential split's arm flag and index. - "household.household_is_cgt_residential_clone", - "household.cgt_residential_clone_index", "household.household_clone_index", - # microcosm#1114: the three area codes leave every artifact under the - # consumers' names (geography_ladder.UK_EXPORT_AREA_CODE_COLUMNS); the - # ladder names never reach an artifact and uk_export_candidate_columns - # translates them before this list is compared. "household.constituency_code_oa", - "household.consumer_debt", - # microcosm#932: the five nation-native aliases of the atomic assignment - # (output_area_code, data_zone_code, intermediate_zone_code, - # super_data_zone_code, district_electoral_area_code) never reach an - # artifact: graph_terminal._tables drops them at the single-year export - # boundary. They are listed because the full-build export-surface gate - # reads the graph frame's in-memory columns (full_gates.py, - # ``parity_evidence.candidate_columns``) before that boundary, and would - # otherwise flag them as unreviewed extras (the F8 gap on #932). "household.data_zone_code", "household.district_electoral_area_code", - "household.electricity_consumption", - "household.electricity_kwh", - "household.gas_consumption", - "household.gas_kwh", "household.geography_household_key", - "household.has_fuel_consumption", - # The LCFS prepayment-meter flag (microcosm#1110): policyengine-uk reads it - # from 2.123.3 for the Energy Price Guarantee prepayment discount, and an - # older engine ignores the column; the incumbent does not carry it yet - # (uk-data#539). - "household.uses_energy_prepayment_meter", - # microcosm#1359: the LCFS excise step's policyengine-uk inputs (the - # alcohol and tobacco ones from 2.145.0, the gambling duty bases from - # 2.151.0); an older engine ignores a column it does not know, and the - # incumbent carries none of them. - "household.beer_litres", - "household.cider_litres", - "household.wine_litres", - "household.spirits_litres", - "household.other_fermented_litres", - "household.beer_draught_share", - "household.cider_draught_share", - "household.cigarettes_per_week", - "household.hand_rolling_tobacco_grams_per_week", - "household.cigars_grams_per_week", - "household.other_tobacco_grams_per_week", - "household.remote_gaming_net_losses", - "household.pool_betting_net_receipts", - "household.bingo_net_receipts", - "household.national_lottery_ticket_spending", - "household.land_based_casino_net_losses", - "household.machine_games_lower_rate_net_takings", - "household.machine_games_standard_rate_net_takings", - "household.machine_games_higher_rate_net_takings", - "household.land_based_fixed_odds_betting_net_losses", - "household.remote_uk_horseracing_betting_net_losses", - "household.remote_other_betting_net_losses", - "household.betting_exchange_commission", - "household.horse_and_dog_pool_betting_net_losses", - "household.household_is_capital_gains_clone", - "household.household_is_cgt_support_copy", - "household.household_is_spi_income_band_donor", - "household.household_is_spi_synthetic", - # #1003: the WAS Lifetime ISA stage's household total and person cells. - "household.household_lifetime_isa_balance", - # #930: the NTS bus-travel stage's household journey cell. - "household.household_local_bus_trips", - # microcosm#1418: the dense build's Rural Fuel Duty Relief flag, set off - # each placed household's output area; the incumbent does not carry it. "household.in_rural_fuel_duty_relief_area", - "household.spi_income_band_donor_lower_bound", "household.intermediate_zone_code", "household.la_code_oa", - # #953: the engine's household local_authority enum input, written by the - # rowwise geography ladder from local_authority_code. The incumbent never - # carried it, so the coverage manifest cannot list it; this allow-list - # binds at the national release-cut export gate, and the rowwise lane's - # own guard is the ladder gate's code/member consistency check. "household.local_authority", "household.lsoa_code", - "household.mortgage_debt", "household.msoa_code", - "household.non_residential_property_purchased", - "household.num_vehicles", "household.oa_code", - "household.ons_household_type", "household.output_area_code", - "household.private_pension_wealth", - "household.property_purchased", - "household.rail_usage", "household.region_code_oa", "household.stocks_and_shares_isa", "household.super_data_zone_code", - "person.a_and_e_visits", - "person.aa_category", - "person.admitted_patient_visits", - "person.age_left_full_time_education", - "person.age_started_or_accepted_current_education_or_training", - "person.attends_private_school", - "person.attends_private_school_random_draw", - "person.capital_gains_asset_type", - "person.capital_gains_badr", - "person.capital_gains_residential_property", - "person.cgt_residential_probability", - "person.bus_in_london_trips", - "person.bus_pass_eligible", - "person.care_hours", - "person.charitable_investment_gifts", - "person.dla_m_category", - "person.dla_sc_category", - "person.employment_sector", - "person.contributory_benefits_annual_share", - "person.employment_income_annual_share", - "person.employment_income_other_spells", - "person.private_pension_income_annual_share", - "person.self_employment_income_annual_share", - "person.self_employment_income_other_spells", - # The self-employed's trading allowance claim status, which the enhanced - # FRS leaves at the engine's UNKNOWN, and the full-relief group's carrier, - # which the bound SPI self-employment rows read (microcosm#1439). - "person.trading_allowance_claim_status", - "person.trading_allowance_full_relief", - # The annuity and trust part of private_pension_income, a calibration-only - # descriptor no engine input reads (microcosm#1428). - "person.annuity_and_trust_income", - "person.esa_health_condition_proxy", - "person.esa_includes_support_component", - "person.esa_support_group_proxy", - "person.gift_aid", - "person.has_lifetime_isa", - "person.highest_education", - "person.is_before_universal_credit_qualifying_young_person_terminal_date", - "person.is_blind", - "person.is_claimant_or_partner", - "person.is_engaged_in_owned_company_trade", - "person.is_hbai_dependent_child", - "person.is_in_full_time_education", - "person.is_in_non_advanced_education", - "person.is_parent", - "person.is_uc_claimant", - "person.legacy_jobseeker_proxy", - "person.lifetime_isa_balance", - "person.lifetime_isa_payments", - "person.cash_isa_balance", - "person.stocks_and_shares_isa_balance", - "person.cash_isa_subscriptions", - "person.stocks_and_shares_isa_subscriptions", - # policyengine-uk 2.165.0's company car inputs, which the company_cars stage - # writes (PolicyEngine/uk-autumn-budget-dashboard-2026#316). - "person.company_car_list_price", - "person.company_car_co2_emissions", - "person.company_car_fuel_type", - "person.company_car_electric_range", - "person.company_car_capital_contributions", - "person.company_car_private_fuel_provided", - # policyengine-uk's vehicle excise duty inputs for households' own cars, - # which the lcfs_consumption stage's NTS car step writes (microcosm#1095 B3). - "person.car_fuel_type", - "person.car_co2_emissions", - "person.car_engine_size", - "person.car_first_registration_date", - "person.car_is_ved_exempt", - "person.individual_savings_account_interest_income", - "person.local_bus_single_fare_share", - "person.local_bus_trips", - "person.local_bus_use_band", - "person.loss_relief", - "person.ni_class_4_losses_brought_forward", - "person.ons_family_index", - "person.ons_family_role", - "person.other_local_bus_trips", - "person.outpatient_visits", - "person.owned_company_carries_on_property_business", - "person.owned_company_carries_on_trade", - "person.owned_company_income_share", - "person.owned_company_is_main_employment", - "person.pension_contributions_via_salary_sacrifice", - "person.pip_dl_points", - "person.pip_m_points", - "person.property_finance_costs", - "person.property_rental_income", - "person.receives_benefits_in_own_right", - "person.rent_paid_as_boarder", - "person.rent_paid_as_lodger", - "person.relationship_to_head", - "person.reports_outstanding_student_loan", - "person.salary_sacrifice_asked", - "person.salary_sacrifice_reported", - "person.sic_industry_division", - "person.soc2020_major_group", - "person.soc2020_sub_major_group", - "person.stands_as_sole_owner_or_partner_of_company", - "person.statutory_paternity_pay", - "person.student_loan_balance", - "person.student_loan_plan", - "person.tax_free_childcare_spend_routed_share", - "person.is_apprentice", - "person.trading_loss", - "person.uc_is_in_startup_period", - "person.uc_is_in_gainful_self_employment", - # microcosm#1095 (Y1): the limited capability flags the - # uc_limited_capability stage stores, DWP's status on linked claims and - # the engine's proxy elsewhere. - "person.uc_limited_capability_for_WRA", - "person.uc_limited_capability_for_work", - "person.is_looked_after_by_local_authority", - "person.child_limit_non_parental_caring_exception", - "person.would_claim_carers_allowance", - "person.would_claim_marriage_allowance", - "person.would_claim_scp", - "person.max_free_entitlement_hours_used", - "person.person_is_spi_income_band_carrier", - "person.person_is_spi_income_recipient", - # microcosm#1063 c9 (ruling 2026-10-02): every declared stage output the - # enhanced-FRS incumbent never carried is allow-listed rather than - # dropped, so the release candidate ships the spine's own surface. The - # certifier rehearsal R5 on the 2026-09-30 build named them: the source - # and support-channel lineage of the three entities, the council tax - # family (#934), the reported-benefit inputs the take-up stages read - # (the five internal disability carriers are not among them: they leave - # at the release boundary, UK_RELEASE_EXPORT_DROPPED_COLUMNS), - # the SPI channel's hmrc_spi_* leaves (#717), the FRS education and - # housing fields, the two raw FRS codes the spine fences - # (ossben_identifiable_subset, srp_regular_code5) and - # other_investment_income. Only incapacity_benefit_reported, an all-zero - # layer the pinned reference leaves off its surface, is dropped - # (national_frame.UK_RELEASE_EXPORT_DROPPED_COLUMNS). - "benunit.benunit_source_id", - "benunit.benunit_support_channel", - "benunit.benunit_support_clone_index", - "benunit.dependent_children", - "household.council_tax_rebate", - "household.council_tax_reported", - "household.council_tax_single_adult_raw", - "household.household_source_id", - "household.household_support_channel", - "household.household_support_clone_index", - "household.num_bedrooms", - "household.source_household_id", - "household.source_household_key", - "household.source_year", - "household.subrent", - "person.disabled_students_allowance_eligible_expenses", - "person.free_school_breakfasts", - "person.hmrc_spi_assessable_income", - "person.hmrc_spi_employed_income", - "person.hmrc_spi_employment_benefits", - "person.hmrc_spi_employment_expenses", - "person.hmrc_spi_incapacity_benefit_income", - "person.hmrc_spi_miscellaneous_employment_income", - "person.hmrc_spi_other_income", - "person.hmrc_spi_other_social_security_income", - "person.hmrc_spi_pay", - "person.hmrc_spi_state_pension_income", - "person.hmrc_spi_taxable_termination_pay", - "person.hmrc_spi_total_earned_income", - "person.hmrc_spi_total_investment_income", - "person.hmrc_spi_unemployment_benefit_income", - "person.is_in_approved_training", - "person.ossben_identifiable_subset", - "person.other_investment_income", - "person.person_source_id", - "person.person_support_channel", - "person.person_support_clone_index", - "person.srp_regular_code5", "benunit.benunit_clone_index", "person.person_clone_index", - # microcosm#1194: the build identifiers the release keeps beside the - # renumbered ones (national_frame.UK_BUILD_IDENTIFIER_COLUMNS). The - # lineage columns above describe them. - "benunit.benunit_build_id", - "household.household_build_id", - "person.person_build_id", - # The months since each person's last birthday, stored by the root stage - # so the engine reads one date of birth everywhere (microcosm#1095). - "person.months_since_last_birthday", "household.itl1_code", "household.itl2_code", "household.itl3_code", "household.ward_code", - # War Widow's or Widower's Pension, split out of bsp_reported (#1222, - # pe-uk#2382). The enhanced FRS still adds FRS benefit code 9 into - # bsp_reported, so its surface has no such input. - "person.war_widows_pension_reported", - # Widowed Parent's Allowance, split out of FRS code 6 by rate (#1222, - # pe-uk#2572). The enhanced FRS keeps the whole code in bsp_reported. - "person.widowed_parents_allowance_reported", - # Spousal maintenance (ALIAMT or ALUAMT, and APAMT; #1320, pe-uk#2572). - # The enhanced FRS reads no alimony and keeps APAMT a private transfer. - "person.spousal_maintenance_income", - # Bonus sacrifice (#1340, pe-uk#2572 section 1): the enhanced FRS carries - # salary sacrifice only. - "person.pension_contributions_via_bonus_sacrifice", - # Liquid and solid heating fuels (#1278, pe-uk#2572): the enhanced FRS - # draws no fuel beside electricity and gas. - "household.liquid_fuel_consumption", - "household.solid_fuel_consumption", - # The share split inside corporate_wealth (pe-uk#1974) and the debt - # secured on other property (UC Regs 2013 reg 49(1)(b)), from the WAS - # property draw (microcosm#1095): the enhanced FRS carries corporate_wealth - # alone and no secured debt. - "household.directly_held_shares", - "household.unit_and_investment_trusts", - "household.other_residential_property_secured_debt", - "household.non_residential_property_secured_debt", ) UK_KNOWN_MISSING_REFERENCE_EXPORT_COLUMNS: tuple[str, ...] = ( @@ -598,8 +294,8 @@ def uk_export_candidate_columns(frame: Any) -> set[str]: Structural id columns are not exported content (microcosm#1063 c9: the certifier rehearsal listed every id as an unreviewed extra), and the frame's household weights live beside the tables, so the surface carries - ``household.household_weight`` explicitly, as the enhanced-FRS reference - does. + ``household.household_weight`` explicitly, as the reference build's + surface does (the reference is measured with this function too). """ from microcosm.build.uk_runtime.geography_ladder import ( @@ -1083,8 +779,13 @@ def uk_export_surface_gate( *, allowed_extra_columns: Iterable[str] = UK_ALLOWED_EXTRA_EXPORT_COLUMNS, reviewed_exclusions: Mapping[str, str] | None = None, + reference_name: str = UK_REFERENCE_DATASET_NAME, ) -> GateResult: - """Run the incumbent-compatible UK export-surface gate.""" + """Compare the candidate's export surface with the reference build's. + + ``reference_name`` labels the reference in the report: the binding + passes the build the cut's reference step recorded. + """ candidate = {str(name) for name in candidate_columns} reference = {str(name) for name in reference_columns} @@ -1093,7 +794,7 @@ def uk_export_surface_gate( candidate, reference, candidate_name=UK_CANDIDATE_DATASET_NAME, - reference_name=UK_REFERENCE_DATASET_NAME, + reference_name=reference_name, allowed_extra_columns=allowed_extra_columns, reviewed_exclusions=exclusions, ) @@ -1102,7 +803,7 @@ def uk_export_surface_gate( if not candidate: failures.append("UK candidate export-surface evidence is empty.") if not reference: - failures.append("Enhanced-FRS reference export-surface evidence is empty.") + failures.append("The reference build's export-surface evidence is empty.") if forbidden: failures.append( f"{UK_CANDIDATE_DATASET_NAME}: exports {len(forbidden)} reviewed " @@ -1112,7 +813,15 @@ def uk_export_surface_gate( name=result.name, passed=not failures, failures=tuple(failures), - details={**dict(result.details), "forbidden_candidate_columns": forbidden}, + details={ + **dict(result.details), + "forbidden_candidate_columns": forbidden, + # Declared extras the reference now carries: reported, not failed, + # since a promotion moves the reference past them (microcosm#1095). + "redundant_allowed_extra_columns": sorted( + {str(name) for name in allowed_extra_columns} & reference + ), + }, ) @@ -1120,7 +829,7 @@ def uk_target_surface_gate( candidate_targets: Iterable[str], reference_targets: Iterable[str], ) -> GateResult: - """Require the UK candidate target surface to cover enhanced FRS.""" + """Require the UK candidate target surface to cover the reference register.""" candidate = {str(name) for name in candidate_targets} reference = {str(name) for name in reference_targets} @@ -1128,13 +837,13 @@ def uk_target_surface_gate( candidate, reference, candidate_name=UK_CANDIDATE_DATASET_NAME, - reference_name=UK_REFERENCE_DATASET_NAME, + reference_name="reference_register", ) failures = [*result.failures] if not candidate: failures.append("UK candidate target-surface evidence is empty.") if not reference: - failures.append("Enhanced-FRS reference target-surface evidence is empty.") + failures.append("Reference register target-surface evidence is empty.") return GateResult( name=result.name, passed=not failures, diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py index da7693d0bc..5f10f7addf 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py @@ -13,22 +13,28 @@ def test_full_required_set_with_signal_passes(self) -> None: } ) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert result.passed assert result.failures == () assert result.details["dormant_exclusions"] == ["property_income"] - def test_missing_required_column_fails_and_names_efrs(self) -> None: + def test_missing_required_column_fails_and_names_the_reference(self) -> None: frame = _person_frame({"employment_income": np.asarray([0.0, 52_000.0])}) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert not result.passed assert result.details["missing"] == ["dividend_income"] assert any( "dividend_income" in failure - and "enhanced-FRS input column is absent" in failure + and "reference input column is absent" in failure for failure in result.failures ) @@ -41,16 +47,19 @@ def test_formula_owned_persisted_override_is_a_hard_requirement(self) -> None: absent = uk_release_input_coverage_gate( _person_frame({"age": np.asarray([40, 70])}), engine, + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) default_only = uk_release_input_coverage_gate( _person_frame({"state_pension_reported": np.asarray([0.0, 0.0])}), engine, + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) populated = uk_release_input_coverage_gate( _person_frame({"state_pension_reported": np.asarray([0.0, 12_000.0])}), engine, + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -66,7 +75,10 @@ def test_default_only_required_column_fails(self) -> None: } ) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert not result.passed assert result.details["degenerate_required"] == ["dividend_income"] @@ -79,6 +91,7 @@ def test_required_column_on_wrong_entity_fails(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, manifest=_CONTRACT, ) @@ -109,7 +122,10 @@ def test_required_column_without_valid_observations_fails(self, values) -> None: } ) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert not result.passed assert result.details["degenerate_required"] == ["dividend_income"] @@ -123,7 +139,10 @@ def test_stale_reviewed_exclusion_fails(self) -> None: } ) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert not result.passed assert result.details["stale_exclusions"] == ["property_income"] @@ -140,7 +159,10 @@ def test_absent_or_default_only_exclusion_passes(self) -> None: } ) result = uk_release_input_coverage_gate( - frame, _StubEngine(_DEFAULTS), manifest=_CONTRACT + frame, + _StubEngine(_DEFAULTS), + reference_surface=NO_REFERENCE_INPUTS, + manifest=_CONTRACT, ) assert result.passed assert result.details["reviewed_exclusions"] == { @@ -159,6 +181,7 @@ def test_signal_only_on_zero_weight_rows_fails_effective_mass(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -180,6 +203,7 @@ def test_positive_but_dust_mass_below_reviewed_floor_fails(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -201,6 +225,7 @@ def test_signal_above_reviewed_effective_mass_floor_passes(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -240,6 +265,7 @@ def test_explicit_person_weights_cannot_override_household_mass(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=_manifest((UKReleaseInputColumn("gift_aid", "required"),)), ) @@ -267,6 +293,7 @@ def test_base_signal_cannot_satisfy_spi_distributional_family(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -289,6 +316,7 @@ def test_positive_spi_signal_satisfies_distributional_family(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -320,6 +348,7 @@ def test_family_requires_reviewed_weight_kind_and_mass_record(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -360,6 +389,7 @@ def test_family_accepts_reviewed_calibrated_mass_state(self) -> None: result = uk_release_input_coverage_gate( frame, _StubEngine({"gift_aid": 0.0}), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -381,6 +411,7 @@ def test_integer_encoded_enum_default_is_not_signal(self) -> None: result = uk_release_input_coverage_gate( frame, PolicyEngineUKCoverageEngine(), + reference_surface=NO_REFERENCE_INPUTS, manifest=contract, ) @@ -390,13 +421,17 @@ def test_integer_encoded_enum_default_is_not_signal(self) -> None: def test_zero_mass_signal_does_not_stale_a_reviewed_exclusion(self) -> None: contract = _manifest((_reviewed_gift_aid_exclusion(),)) frame = _weighted_person_frame( - {"gift_aid": np.asarray([900.0, 0.0])}, + { + "gift_aid": np.asarray([900.0, 0.0]), + "employment_income": np.asarray([0.0, 30_000.0]), + }, np.asarray([0.0, 1_000.0]), ) result = uk_release_input_coverage_gate( frame, - _StubEngine({"gift_aid": 0.0}), + _StubEngine({"gift_aid": 0.0, "employment_income": 0.0}), + reference_surface=reference_surface("employment_income"), manifest=contract, ) @@ -416,13 +451,17 @@ def test_deferred_family_does_not_enforce_future_distributional_gate(self) -> No family_coverage=family, ) frame = _weighted_person_frame( - {"gift_aid": np.asarray([900.0, 0.0])}, + { + "gift_aid": np.asarray([900.0, 0.0]), + "employment_income": np.asarray([0.0, 30_000.0]), + }, np.asarray([0.0, 1_000.0]), ) result = uk_release_input_coverage_gate( frame, - _StubEngine({"gift_aid": 0.0}), + _StubEngine({"gift_aid": 0.0, "employment_income": 0.0}), + reference_surface=reference_surface("employment_income"), manifest=contract, ) @@ -437,9 +476,10 @@ def test_shipped_manifest_is_current(self) -> None: assert_uk_release_input_coverage_manifest_current( engine=_StubEngine({}, set(manifest.declared_columns)) ) - # microcosm#1106: property_wealth is formula-owned in policyengine-uk, - # so the release leaves it to the engine; it is an engine-derived - # exclusion, not parity debt, and its components stay required. So is + # microcosm#1095: the required inputs come from the cut's reference + # build, so the manifest declares only the reviewed exclusions. + # property_wealth is formula-owned in policyengine-uk, so the release + # leaves it to the engine (microcosm#1106); so is # tax_free_savings_income under the additive ISA income contract # (microcosm#1360), and so are the PIP categories, which the engine # derives from the points the release stores (microcosm#1095, P1). @@ -449,15 +489,14 @@ def test_shipped_manifest_is_current(self) -> None: "property_wealth", "tax_free_savings_income", ] - assert manifest.required_columns == ( - frozenset(load_efrs_parity_reference().populated_layers) - - set(engine_derived) - ) + assert manifest.required_columns == frozenset() assert set(manifest.reviewed_exclusions) == set(engine_derived) assert load_efrs_parity_known_gaps() == () assert [ gap.variable for gap in load_efrs_engine_derived_exclusions() ] == engine_derived + assert manifest.reference["kind"] == "current_build" + assert manifest.schema_version == 4 assert manifest.required_build_stages == frozenset( { "frs_hmrc_spine_leaves", @@ -496,24 +535,29 @@ def test_shipped_manifest_is_current(self) -> None: {"charitable_investment_gifts", "gift_aid"} ) - def test_manifest_refuses_empty_columns(self, tmp_path) -> None: - bad = tmp_path / "empty.json" - bad.write_text( - json.dumps( - { - "reference": {}, - "candidate_evidence": {"tier": "frs"}, - "effective_mass_coverage": { - "weight_source": "household_weight", - "minimum_nondefault_mass_share": 1e-6, - "reviewed_on": "2026-07-11", - "rationale": "reviewed test floor", - }, - "columns": {}, - } - ) - ) - with pytest.raises(ValueError, match="vacuous"): + def test_manifest_refuses_another_schema_or_reference(self, tmp_path) -> None: + payload = { + "schema_version": 4, + "reference": {"kind": "current_build"}, + "candidate_evidence": {"tier": "frs"}, + "effective_mass_coverage": { + "weight_source": "household_weight", + "minimum_nondefault_mass_share": 1e-6, + "reviewed_on": "2026-07-11", + "rationale": "reviewed test floor", + }, + "columns": {}, + } + bad = tmp_path / "manifest.json" + # No declared column is fine: the reference supplies the required + # inputs, and the gate refuses an empty required set itself. + bad.write_text(json.dumps(payload)) + assert load_uk_release_input_coverage_manifest(str(bad)).columns == () + bad.write_text(json.dumps({**payload, "schema_version": 3})) + with pytest.raises(ValueError, match="schema_version must be 4"): + load_uk_release_input_coverage_manifest(str(bad)) + bad.write_text(json.dumps({**payload, "reference": {"filename": "x.h5"}})) + with pytest.raises(ValueError, match="reference.kind must be"): load_uk_release_input_coverage_manifest(str(bad)) def test_distributional_family_requires_channel_denominator(self, tmp_path) -> None: @@ -676,41 +720,41 @@ def test_reviewed_exclusion_requires_tracking_note(self) -> None: def test_engine_graph_drift_is_rejected(self) -> None: manifest = load_uk_release_input_coverage_manifest() - graph = set(manifest.declared_columns) - {"employment_income"} - with pytest.raises(ValueError, match="employment_income"): + graph = set(manifest.declared_columns) - {"property_wealth"} + with pytest.raises(ValueError, match="property_wealth"): assert_uk_release_input_coverage_manifest_current( engine=_StubEngine({}, graph) ) - def test_engine_entity_drift_is_rejected(self) -> None: + def test_unresolvable_owning_entity_is_rejected(self) -> None: manifest = load_uk_release_input_coverage_manifest() - graph = set(manifest.declared_columns) - with pytest.raises(ValueError, match="owning entities disagree"): + + class _NoEntities(_StubEngine): + def variable_entities(self, names): + return {} + + with pytest.raises(ValueError, match="property_wealth"): assert_uk_release_input_coverage_manifest_current( - engine=_StubEngine( - {}, - graph, - entities={"employment_income": "household"}, - ) + engine=_NoEntities({}, set(manifest.declared_columns)) ) - def test_manifest_cannot_demote_a_column_absent_from_known_gaps(self) -> None: + def test_manifest_cannot_exclude_a_restored_input(self) -> None: manifest = load_uk_release_input_coverage_manifest() - columns = tuple( - replace( - column, - status="reviewed_exclusion", - reason="not yet ported from enhanced FRS pipeline — pending review", - tracking_note="Tracked in UK_COVERAGE_PROGRESS.md.", - ) - if column.name == "employment_income" - else column - for column in manifest.columns + demoted = replace( + manifest, + columns=( + *manifest.columns, + UKReleaseInputColumn( + "charitable_investment_gifts", + "reviewed_exclusion", + reason="not yet ported from enhanced FRS pipeline — pending review", + tracking_note="Tracked in UK_COVERAGE_PROGRESS.md.", + ), + ), ) - demoted = replace(manifest, columns=columns) - with pytest.raises(ValueError, match="employment_income.*remain required"): + with pytest.raises(ValueError, match="charitable_investment_gifts.*restored"): assert_uk_release_input_coverage_manifest_current( - engine=_StubEngine({}, set(manifest.declared_columns)), + engine=_StubEngine({}, set(demoted.declared_columns)), manifest=demoted, ) @@ -755,71 +799,59 @@ def test_deferred_family_columns_must_remain_reviewed_exclusions(self) -> None: manifest=replace(manifest, family_coverage=families), ) - def test_promoted_family_columns_must_be_required(self) -> None: + def test_promoted_family_columns_cannot_be_excluded(self) -> None: manifest = load_uk_release_input_coverage_manifest() - columns = tuple( - replace( - column, - status="reviewed_exclusion", - reason="not yet ported from enhanced FRS pipeline — pending review", - tracking_note="Tracked in UK_COVERAGE_PROGRESS.md.", - ) - if column.name == "gift_aid" - else column - for column in manifest.columns - ) + columns = (*manifest.columns, _reviewed_gift_aid_exclusion()) with pytest.raises(ValueError, match="required_at_build"): assert_uk_release_input_coverage_manifest_current( - engine=_StubEngine({}, set(manifest.declared_columns)), + engine=_StubEngine({}, {*manifest.declared_columns, "gift_aid"}), manifest=replace(manifest, columns=columns), ) - def test_loader_aliases_are_hard_covered(self) -> None: + def test_loader_aliases_are_required_when_the_reference_carries_them( + self, + ) -> None: + # The required inputs come from the reference (microcosm#1095): an + # alias column the reference build carries is required of the + # candidate like any other input, and no alias is ever excluded. manifest = load_uk_release_input_coverage_manifest() - assert set(UK_LOADER_INPUT_ALIASES) <= set(manifest.required_columns) + assert set(UK_LOADER_INPUT_ALIASES).isdisjoint(manifest.reviewed_exclusions) + frame = _person_frame({"employment_income": np.asarray([0.0, 52_000.0])}) + result = uk_release_input_coverage_gate( + frame, + _StubEngine({name: 0.0 for name in UK_LOADER_INPUT_ALIASES}), + reference_surface=reference_surface(*UK_LOADER_INPUT_ALIASES), + manifest=_manifest(()), + ) + assert not result.passed + assert result.details["missing"] == sorted( + set(UK_LOADER_INPUT_ALIASES) - {"employment_income"} + ) + assert result.details["required_from_reference"] == len(UK_LOADER_INPUT_ALIASES) - def test_formula_owned_persisted_overrides_are_hard_covered(self) -> None: + def test_engine_derived_exclusions_are_the_only_excluded_overrides(self) -> None: manifest = load_uk_release_input_coverage_manifest() - reference = json.loads( - ( - _REPO_ROOT - / "packages" - / "microcosm-build" - / "src" - / "microcosm" - / "build" - / "uk" - / "efrs_parity_reference.json" - ).read_text(encoding="utf-8") - ) - overrides = set( - reference["engine"]["formula_owned_persisted_overrides_included"] - ) - assert len(overrides) == 13 # The exceptions are declared: property_wealth, whose persisted copy # overrides the engine's component sum and is never uprated, so the # release leaves it to the formula (microcosm#1106), and # tax_free_savings_income, the engine's sum of the ISA interest the # additive contract stores apart (microcosm#1360). Every other - # formula-owned override stays required. + # formula-owned override the reference carries is required. # - # The PIP categories are engine-derived since the reference: its - # engine (2.89.0) read them as plain inputs, so they are not among its - # overrides, but the pinned engine derives them from the points the - # release stores (microcosm#1095, P1). + # The PIP categories are engine-derived since the enhanced FRS: its + # engine (2.89.0) read them as plain inputs, but the pinned engine + # derives them from the points the release stores (microcosm#1095, P1). since_reference = {"pip_dl_category", "pip_m_category"} engine_derived = {gap.variable for gap in load_efrs_engine_derived_exclusions()} - expected = {"property_wealth", "tax_free_savings_income"} | since_reference - assert engine_derived == expected - assert since_reference.isdisjoint(overrides) - assert engine_derived - since_reference <= overrides + assert engine_derived == ( + {"property_wealth", "tax_free_savings_income"} | since_reference + ) from policyengine_uk import CountryTaxBenefitSystem variables = CountryTaxBenefitSystem().variables assert not any(variables[name].is_input_variable() for name in since_reference) - assert overrides - engine_derived <= set(manifest.required_columns) - assert engine_derived.isdisjoint(manifest.required_columns) + assert set(manifest.reviewed_exclusions) == engine_derived def test_live_uk_adapter_recognises_loader_aliases(self) -> None: engine = PolicyEngineUKCoverageEngine() diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py b/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py index fe9f627028..9466c0cf59 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py @@ -49,23 +49,38 @@ def test_constant_names_every_weight_share_formula_and_its_engine_inputs(system) def test_named_columns_are_release_surface_household_columns(): - """The columns the check reads leave with the release: every one is an - enhanced-FRS household input (the parity reference) or a reviewed extra - household column of the export surface (the allow-list).""" - from microcosm.build.uk_runtime import load_efrs_parity_reference - from microcosm.build.uk_runtime.terminal_gates import ( - UK_ALLOWED_EXTRA_EXPORT_COLUMNS, + """The columns the check reads leave with the release: every one is a + household column the spine declares and the release boundary keeps. + + Until microcosm#1095 this read the enhanced-FRS reference and the export + allow-list; the export surface now compares with the current build, so + the spine's declared surface through the production export path stands + in for both. + """ + import numpy as np + import pandas as pd + + from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph + from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + uk_release_export_frame, ) - reference = load_efrs_parity_reference().input_entities - household = { - column for column, entity in reference.items() if entity == "household" + columns = { + "person": ["person_id", "person_benunit_id", "person_household_id"], + "benunit": ["benunit_id"], + "household": ["household_id"], } - household |= { - name.split(".", 1)[1] - for name in UK_ALLOWED_EXTRA_EXPORT_COLUMNS - if name.startswith("household.") + for cells in uk_spine_endpoint(uk_spine_graph()).inputs: + columns[cells.entity] = list( + dict.fromkeys([*columns[cells.entity], *cells.columns]) + ) + tables = { + entity: pd.DataFrame({column: np.array([1]) for column in names}) + for entity, names in columns.items() } - for formula, columns in UK_WEIGHT_SHARE_FORMULA_INPUTS.items(): - missing = [c for c in columns if c not in household] + frame = uk_national_frame(**tables, time_period="2024", household_weights=[1.0]) + household = set(uk_release_export_frame(frame).table("household").columns) + for formula, columns_read in UK_WEIGHT_SHARE_FORMULA_INPUTS.items(): + missing = [c for c in columns_read if c not in household] assert not missing, f"{formula}: {missing} are not release household columns" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py index 3bbdad4245..0797ad4725 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py @@ -52,6 +52,7 @@ from microcosm.build.uk_runtime.weighted_integrity import UKReviewedExclusion from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import engine_tables +from test_support.microcosm_build.uk_cut_reference import reference_surface from tools.generate_uk_local_target_references import SUPPORT_FLOOR_DEFERRED_AREA_IDS KEY = base64.b64encode(b"\x07" * 32).decode("ascii") @@ -69,6 +70,11 @@ ) +# A reference that carries no input: the coverage tests' manifests declare +# the required inputs themselves (microcosm#1095). +NO_REFERENCE_INPUTS = reference_surface(export_columns=("person.age",)) + + @pytest.fixture(autouse=True) def signing_env(monkeypatch) -> None: monkeypatch.setenv(gate_signing_key_env("uk"), KEY) @@ -79,32 +85,41 @@ def uk_gates(): return load_country_spec("uk").gates -def test_declared_export_surface_preserves_claimants_and_matches_runtime(uk_gates): +def test_declared_export_surface_matches_runtime(uk_gates): from microcosm.build.uk_runtime.terminal_gates import ( UK_ALLOWED_EXTRA_EXPORT_COLUMNS, ) entry = next(entry for entry in uk_gates.gates if entry.id == "uk_export_surface") declared = set(entry.parameters["allowed_extra_columns"]) - assert "person.is_uc_claimant" in declared assert declared == set(UK_ALLOWED_EXTRA_EXPORT_COLUMNS) battery = _run_battery( _tables(), - parity=_parity(candidate_columns={"person.age", "person.is_uc_claimant"}), + parity=_parity(candidate_columns={"person.age", "household.lsoa_code"}), ) outcome = next(o for o in battery.outcomes if o.entry.id == "uk_export_surface") assert outcome.status is GateStatus.PASSED + assert outcome.result.details["reference_name"] == REFERENCE_LABEL + + +def test_export_surface_needs_the_reference_surface(uk_gates): + # Without the cut's reference surface the gate is evidence_absent, which + # blocks a release candidate, never a verdict against no reference. + battery = _run_battery(_tables(), parity=None) + outcome = next(o for o in battery.outcomes if o.entry.id == "uk_export_surface") + assert outcome.status is GateStatus.EVIDENCE_ABSENT -def test_spine_export_surface_is_reference_or_allow_listed(uk_gates): - """Every column the spine declares leaves through the release boundary as - an enhanced-FRS input, an allow-listed extra or a reviewed drop. +def test_spine_export_never_carries_a_reviewed_export_exclusion(uk_gates): + """The release boundary drops every column a candidate must never export. - The export-surface gate runs only at release-cut certification, on a built - file; war_widows_pension_reported (#1222) reached main without an - allow-list entry and blocked the first certification after it. This runs - the same gate on the spine's declared surface through the production - export path, against the checked-in enhanced-FRS reference. + Until microcosm#1095 this ran the export-surface gate on the spine's + declared surface against the checked-in enhanced-FRS reference, so a new + spine column without an allow-list entry failed here before it failed a + certification (#1222). The reference is now the current build of the + line, which the repository does not hold, so that half runs at the cut; + the half that needs no reference stays: a persisted copy of a reviewed + export exclusion overrides the engine's formula. """ from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph @@ -112,12 +127,9 @@ def test_spine_export_surface_is_reference_or_allow_listed(uk_gates): uk_national_frame, uk_release_export_frame, ) - from microcosm.build.uk_runtime.parity_reference import ( - load_efrs_parity_reference, - ) from microcosm.build.uk_runtime.terminal_gates import ( + UK_REVIEWED_EXPORT_EXCLUSIONS, uk_export_candidate_columns, - uk_export_surface_gate, ) columns = { @@ -135,13 +147,7 @@ def test_spine_export_surface_is_reference_or_allow_listed(uk_gates): } frame = uk_national_frame(**tables, time_period="2024", household_weights=[1.0]) candidate = uk_export_candidate_columns(uk_release_export_frame(frame)) - reference = { - f"{entity}.{name}" - for name, entity in load_efrs_parity_reference().input_entities.items() - } - entry = next(entry for entry in uk_gates.gates if entry.id == "uk_export_surface") - result = uk_export_surface_gate(candidate, reference, **dict(entry.parameters)) - assert result.passed, result.failures + assert not candidate & set(UK_REVIEWED_EXPORT_EXCLUSIONS) def _tables(*, n: int = 4, weights=None): @@ -359,6 +365,12 @@ def _run_battery( artifacts["fit_weight_records"] = fit_records if parity is not None: artifacts["parity_evidence"] = parity + # The export-surface gate reads the reference's columns from the + # cut's reference surface (microcosm#1095); the fixture keeps them + # beside the parity evidence for readability. + artifacts["reference_surface"] = reference_surface( + export_columns=tuple(parity.reference_columns), label=REFERENCE_LABEL + ) if armed: artifacts["input_mass_reference"] = _reference() artifacts["aggregate_admin"] = { @@ -1449,6 +1461,7 @@ def test_terminal_mode_threads_frame_engine_and_manifest(self) -> None: artifacts={ "coverage_engine": engine, "coverage_manifest": manifest, + "reference_surface": NO_REFERENCE_INPUTS, "spine_frame": frame, }, ), @@ -1457,6 +1470,7 @@ def test_terminal_mode_threads_frame_engine_and_manifest(self) -> None: direct = uk_release_input_coverage_gate( _uk_gate_surface(frame), engine, + reference_surface=NO_REFERENCE_INPUTS, manifest=manifest, build_state_frame=_uk_gate_surface(frame), ) @@ -1511,7 +1525,10 @@ def variable_entities(self, names): family_coverage={}, ) raw = uk_release_input_coverage_gate( - _uk_gate_surface(frame), engine, manifest=manifest + _uk_gate_surface(frame), + engine, + reference_surface=NO_REFERENCE_INPUTS, + manifest=manifest, ) result = UK_GATE_REGISTRY["release_input_coverage"].evaluate( EvidenceContext( @@ -1519,6 +1536,7 @@ def variable_entities(self, names): artifacts={ "coverage_engine": engine, "coverage_manifest": manifest, + "reference_surface": NO_REFERENCE_INPUTS, # The spine build state the family half reads (#1115). "spine_frame": frame, }, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py index 671495bb60..90a7a0c673 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_certify_release_cut_command.py @@ -35,6 +35,7 @@ def test_certifier_pins_carry_the_byte_sizes_the_logbook_digest_requires( "build_record.json", "terminal_gates.json", "input_mass_reference.json", + "reference_surface.json", "score_vs_reference.json", "reference.json", ): @@ -78,6 +79,8 @@ def fake_run(args, state): "2" * 64, "--input-mass-reference", str(placeholders["input_mass_reference.json"]), + "--reference-surface", + str(placeholders["reference_surface.json"]), "--score-receipt", str(placeholders["score_vs_reference.json"]), "--reference-record", @@ -152,6 +155,7 @@ def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( for name in ( "terminal_gates.json", "input_mass_reference.json", + "reference_surface.json", "score.json", "reference.json", ): @@ -186,6 +190,8 @@ def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( "2" * 64, "--input-mass-reference", str(tmp_path / "input_mass_reference.json"), + "--reference-surface", + str(tmp_path / "reference_surface.json"), "--score-receipt", str(tmp_path / "score.json"), "--reference-record", @@ -198,3 +204,50 @@ def test_certifier_refuses_a_spine_that_is_not_the_candidates_recorded_parent( str(tmp_path / "release_certification.json"), ] ) + + +def test_certifier_refuses_a_reference_surface_of_another_build(tmp_path: Path): + """The export-surface and coverage gates compare with the build the score + and the input-mass gate compare with (microcosm#1095).""" + + import argparse + + record = tmp_path / "reference.json" + record.write_text( + json.dumps( + { + "release": "microcosm-uk-2024-25-national", + "build": "build-a", + "sha256": "1" * 64, + } + ) + ) + surface = tmp_path / "reference_surface.json" + + def write_surface(build: str, sha256: str) -> None: + surface.write_text( + json.dumps( + { + "schema_version": 1, + "kind": "uk_reference_surface", + "label": f"microcosm-uk-2024-25-national@{build}", + "sha256": sha256, + "minimum_nondefault_mass_share": 1e-6, + "export_columns": ["person.age"], + "populated_inputs": ["age"], + "thin_inputs": [], + } + ) + ) + + args = argparse.Namespace(reference_record=record, reference_surface=surface) + write_surface("build-b", "2" * 64) + with pytest.raises(SystemExit, match="not the cut's reference"): + certify._reference_surface(args) + write_surface("build-a", "2" * 64) + with pytest.raises(SystemExit, match="not the cut's reference"): + certify._reference_surface(args) + write_surface("build-a", "1" * 64) + assert certify._reference_surface(args).label == ( + "microcosm-uk-2024-25-national@build-a" + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_employment.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_employment.py index e673fc6bdb..58fdf4b1c6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_employment.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_employment.py @@ -16,11 +16,11 @@ derive_frs_employment, derive_soc2020_major_group_from_frs, ) +from microcosm.build.uk_runtime.national_frame import UK_RELEASE_EXPORT_DROPPED_COLUMNS from microcosm.build.uk_runtime.occupation_constants import ( UK_FRS_SOC2020_MAJOR_GROUP, UK_SOC2020_NO_OCCUPATION, ) -from microcosm.build.uk_runtime.terminal_gates import UK_ALLOWED_EXTRA_EXPORT_COLUMNS from test_support.paths import paths_for UK_PACKAGE = ( @@ -346,6 +346,7 @@ def test_gates_check_the_occupation_domain_and_export_the_column() -> None: enum = by_id["uk_soc2020_major_group_enum_domain"] assert (enum["gate"], enum["phase"]) == ("enum_domain", "terminal") assert enum["parameters"]["columns"] == ["soc2020_major_group"] - exported = by_id["uk_export_surface"]["parameters"]["allowed_extra_columns"] - assert "person.soc2020_major_group" in exported - assert "person.soc2020_major_group" in UK_ALLOWED_EXTRA_EXPORT_COLUMNS + # The column leaves with the release; the current national build exports + # it, so since microcosm#1095 it is part of the reference surface rather + # than a declared extra of the export-surface gate. + assert "soc2020_major_group" not in UK_RELEASE_EXPORT_DROPPED_COLUMNS["person"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_relationships.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_relationships.py index 15736cf7b1..e19688f6ed 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_relationships.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_relationships.py @@ -799,13 +799,21 @@ def test_gates_declare_the_stage_health_and_enum_gates_and_export_columns() -> N enum = by_id["uk_ons_household_type_enum_domain"] assert (enum["gate"], enum["phase"]) == ("enum_domain", "assembled") assert enum["parameters"]["columns"] == ["ons_household_type"] - exported = set(by_id["uk_export_surface"]["parameters"]["allowed_extra_columns"]) - assert { + # The columns leave with the release; the current national build exports + # them, so since microcosm#1095 they are part of the reference surface + # rather than declared extras of the export-surface gate. + from microcosm.build.uk_runtime.national_frame import ( + UK_RELEASE_EXPORT_DROPPED_COLUMNS, + ) + + for name in ( "person.relationship_to_head", "person.ons_family_role", "person.ons_family_index", "household.ons_household_type", - } <= exported + ): + entity, column = name.split(".") + assert column not in UK_RELEASE_EXPORT_DROPPED_COLUMNS.get(entity, ()) # --- stage health ----------------------------------------------------------- diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py index f9c92ef3c7..22b2cf889c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_gates.py @@ -138,11 +138,6 @@ def evidence(monkeypatch): weight_kind=weights.kind, household_weights=weights.values, ) - monkeypatch.setattr( - runtime, - "load_efrs_parity_reference", - lambda: SimpleNamespace(input_entities={"income": "household"}), - ) monkeypatch.setattr( runtime, "uk_aggregate_admin_totals", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 6f2e20b815..25c72c61c1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -1215,11 +1215,6 @@ def artifact(payload, raw=False): local_authority_code=np.array(["E09000001", "E07000002"]), ), ) - monkeypatch.setattr( - full_gates, - "load_efrs_parity_reference", - lambda: SimpleNamespace(input_entities={}), - ) monkeypatch.setattr( full_gates, "uk_aggregate_admin_totals", lambda frame, gates: ({}, []) ) @@ -1577,3 +1572,20 @@ def test_the_calibrated_gates_refuse_a_stale_terminal_code() -> None: ) with pytest.raises(ValueError, match="terminal-only code"): kernel.run(context) + + +def test_the_dense_gates_refuse_references_of_two_builds() -> None: + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_terminal import _require_one_reference + + mass = SimpleNamespace(label="microcosm-uk-2024-25-dense@a", sha256="1" * 64) + same = SimpleNamespace(label="microcosm-uk-2024-25-dense@a", sha256="1" * 64) + other = SimpleNamespace(label="microcosm-uk-2024-25-dense@b", sha256="2" * 64) + _require_one_reference({"input_mass_reference": mass, "reference_surface": same}) + _require_one_reference({"input_mass_reference": mass}) + _require_one_reference({"reference_surface": other}) + with pytest.raises(ValueError, match="from one reference"): + _require_one_reference( + {"input_mass_reference": mass, "reference_surface": other} + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_uprating_engine_pins.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_uprating_engine_pins.py index 6854455af2..5aac92dfaf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_uprating_engine_pins.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_uprating_engine_pins.py @@ -238,5 +238,6 @@ def test_the_build_and_the_certifier_refuse_an_unpinned_engine(monkeypatch) -> N parity_evidence=None, fit_weight_records=None, input_mass_reference={}, + reference_surface=None, exclusions_evaluated_on=date(2026, 10, 18), ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_reference_surface.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_reference_surface.py new file mode 100644 index 0000000000..14f3d0e94d --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_reference_surface.py @@ -0,0 +1,158 @@ +"""The column surface of a UK cut's reference build (microcosm#1095).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import national_frame, weight_year +from microcosm.build.uk_runtime.cut_reference import ( + UK_CUT_REFERENCE_INPUT_MASS_NAME, + UK_CUT_REFERENCE_SURFACE_NAME, + uk_cut_reference_evidence_done, +) +from microcosm.build.uk_runtime.reference_surface import ( + load_uk_reference_surface, + uk_reference_surface_payload, +) +from test_support.microcosm_build.uk_release_input_coverage import _StubEngine + +LABEL = "microcosm-uk-2024-25-national@uk-frs-calibration-attempt-test" +SHA = "a" * 64 + + +def _payload(**overrides) -> dict: + payload = { + "schema_version": 1, + "kind": "uk_reference_surface", + "label": LABEL, + "sha256": SHA, + "minimum_nondefault_mass_share": 1e-6, + "export_columns": ["person.employment_income", "person.dividend_income"], + "populated_inputs": ["employment_income"], + "thin_inputs": ["dividend_income"], + } + payload.update(overrides) + return payload + + +def test_a_surface_round_trips_names_and_flags(tmp_path: Path) -> None: + path = tmp_path / UK_CUT_REFERENCE_SURFACE_NAME + path.write_text(json.dumps(_payload())) + surface = load_uk_reference_surface(path) + assert surface.label == LABEL + assert surface.sha256 == SHA + assert surface.minimum_nondefault_mass_share == 1e-6 + assert surface.export_columns == { + "person.employment_income", + "person.dividend_income", + } + assert surface.carried_inputs == {"employment_income", "dividend_income"} + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"schema_version": 2}, "schema-1"), + ({"kind": "uk_input_mass_reference"}, "schema-1"), + ({"label": "no-build"}, "@"), + ({"sha256": "short"}, "sha256"), + ({"minimum_nondefault_mass_share": 0.0}, "effective-mass floor"), + ({"minimum_nondefault_mass_share": True}, "effective-mass floor"), + ({"populated_inputs": "employment_income"}, "list of column names"), + ({"thin_inputs": ["employment_income"]}, "both populated and thin"), + ({"export_columns": []}, "exports no column"), + ], +) +def test_a_malformed_surface_is_refused(tmp_path: Path, overrides, message) -> None: + path = tmp_path / UK_CUT_REFERENCE_SURFACE_NAME + path.write_text(json.dumps(_payload(**overrides))) + with pytest.raises(ValueError, match=message): + load_uk_reference_surface(path) + + +def _reference_frame(): + # Five households: two clones of one source household (101), two more + # sources, and a zero-weight row. Employment income sits on three + # sources, dividends on one (both clones of 101), savings interest only + # on the zero-weight row. + household = pd.DataFrame( + { + "household_id": np.arange(1, 6, dtype="int64"), + "household_source_id": np.asarray([101, 101, 102, 103, 104]), + } + ) + person = pd.DataFrame( + { + "person_id": np.arange(1, 6, dtype="int64"), + "person_household_id": np.arange(1, 6, dtype="int64"), + "person_benunit_id": np.arange(1, 6, dtype="int64"), + "employment_income": np.asarray([1.0, 1.0, 1.0, 1.0, 0.0]), + "dividend_income": np.asarray([5.0, 5.0, 0.0, 0.0, 0.0]), + "savings_interest_income": np.asarray([0.0, 0.0, 0.0, 0.0, 9.0]), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(1, 6, dtype="int64")}) + return national_frame.uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + household_weights=np.asarray([10.0, 10.0, 10.0, 10.0, 0.0]), + ) + + +def test_the_payload_flags_inputs_by_how_many_source_households_carry_them( + monkeypatch, +) -> None: + frame = _reference_frame() + monkeypatch.setattr( + national_frame, "load_uk_national_frame", lambda path: (frame, None) + ) + # The test frame records no solve; the real one is put back at its own + # solved weights by the factor its mass log records. + monkeypatch.setattr(weight_year, "uk_recorded_solve_year_frame", lambda f: f) + engine = _StubEngine( + { + "employment_income": 0.0, + "dividend_income": 0.0, + "savings_interest_income": 0.0, + } + ) + + payload = uk_reference_surface_payload( + "reference.h5", label=LABEL, sha256=SHA, engine=engine + ) + + assert payload["label"] == LABEL + assert payload["sha256"] == SHA + assert payload["minimum_nondefault_mass_share"] == 1e-6 + assert payload["populated_inputs"] == ["employment_income"] + # One source household carries dividends: listed by name only, "<3". + assert payload["thin_inputs"] == ["dividend_income"] + assert "person.savings_interest_income" in payload["export_columns"] + assert "person.employment_income" in payload["export_columns"] + # Names and flags only: no figure besides the policy floor. + assert set(payload) == set(_payload()) + + +def test_the_evidence_is_done_only_when_both_files_name_the_build( + tmp_path: Path, +) -> None: + reference = type("Reference", (), {"label": LABEL, "sha256": SHA})() + mass = tmp_path / UK_CUT_REFERENCE_INPUT_MASS_NAME + mass.write_text(json.dumps({"label": LABEL, "identity": {"sha256": SHA}})) + done, reason = uk_cut_reference_evidence_done(reference, tmp_path) + assert not done and UK_CUT_REFERENCE_SURFACE_NAME in reason + + surface = tmp_path / UK_CUT_REFERENCE_SURFACE_NAME + surface.write_text(json.dumps(_payload(sha256="b" * 64))) + done, reason = uk_cut_reference_evidence_done(reference, tmp_path) + assert not done and "another build" in reason + + surface.write_text(json.dumps(_payload())) + assert uk_cut_reference_evidence_done(reference, tmp_path)[0] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py index 9a48a15dc0..37cce14195 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_certification.py @@ -127,22 +127,20 @@ def table(self, entity): # pragma: no cover - never reached class _Registry: specs = () - class _Reference: - input_entities = {} - with pytest.raises(UKReleaseCertificationError, match="name@period"): uk_release_parity_evidence( _Frame(), diagnostics_targets=[{"name": "dwp.uc.households", "relative_error": 0.1}], reference_registry=_Registry(), - parity_reference=_Reference(), ) evidence = uk_release_parity_evidence( _Frame(), diagnostics_targets=[{"name": "dwp.uc.households@2025", "relative_error": 0.1}], reference_registry=_Registry(), - parity_reference=_Reference(), ) + # The export-surface gate reads the reference's columns from the cut's + # reference surface itself (microcosm#1095). + assert not hasattr(evidence, "reference_columns") assert evidence.candidate_targets == {"dwp.uc.households@2025"} assert evidence.target_relative_errors == {"dwp.uc.households@2025": 0.1} # A run built before María's ruling of 2026-10-26 dropped the withholding @@ -161,7 +159,6 @@ class _Reference: }, ], reference_registry=_Registry(), - parity_reference=_Reference(), ) @@ -587,6 +584,7 @@ def test_release_cut_battery_refuses_a_compiled_period_with_no_unsupported_repor parity_evidence=object(), fit_weight_records=None, input_mass_reference={}, + reference_surface=object(), exclusions_evaluated_on=date(2026, 8, 27), gate_registry=_stub_registry(), spine_frame=object(), @@ -614,6 +612,7 @@ def test_release_cut_battery_runs_and_signs(tmp_path: Path, monkeypatch): parity_evidence=SimpleNamespace(), fit_weight_records=None, input_mass_reference={}, + reference_surface=SimpleNamespace(), exclusions_evaluated_on=date(2026, 8, 27), gate_registry=_stub_registry(), spine_frame=object(), diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage.py index 3775dc4696..99c49134d9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage.py @@ -7,6 +7,7 @@ UK_FAMILY_MASS_CHANGE_SEMANTICS, _parse_family_coverage, _valid_mass_record, + uk_effective_input_signal, ) from test_support.microcosm_build.uk_release_input_coverage import * @@ -96,7 +97,10 @@ def test_gate_reads_build_state_from_the_spine_frame(self) -> None: engine = _StubEngine({"gift_aid": 0.0}) on_release = uk_release_input_coverage_gate( - release, engine, manifest=self._contract() + release, + engine, + reference_surface=NO_REFERENCE_INPUTS, + manifest=self._contract(), ) assert not on_release.passed assert any( @@ -105,7 +109,11 @@ def test_gate_reads_build_state_from_the_spine_frame(self) -> None: assert on_release.details["family_build_state_frame"] == "release" result = uk_release_input_coverage_gate( - release, engine, manifest=self._contract(), build_state_frame=spine + release, + engine, + reference_surface=NO_REFERENCE_INPUTS, + manifest=self._contract(), + build_state_frame=spine, ) assert result.passed, result.failures assert result.details["family_build_state_frame"] == "spine" @@ -127,6 +135,7 @@ def test_binding_hands_the_spine_frame_to_the_build_state_half(self) -> None: artifacts = { "coverage_engine": _StubEngine({"gift_aid": 0.0}), "coverage_manifest": self._contract(), + "reference_surface": NO_REFERENCE_INPUTS, "spine_frame": spine, } result = _evaluate_release_input_coverage( @@ -150,6 +159,7 @@ def test_binding_refuses_to_judge_the_calibrated_frame_without_build_state( artifacts = { "coverage_engine": _StubEngine({"gift_aid": 0.0}), "coverage_manifest": self._contract(), + "reference_surface": NO_REFERENCE_INPUTS, } with pytest.raises(ValueError, match="spine build state"): _evaluate_release_input_coverage( @@ -205,3 +215,127 @@ def test_a_record_that_adds_mass_never_satisfies_a_family(self) -> None: ) ) assert not _valid_mass_record({**conserving.__dict__, "entity": "person"}) + + +class TestReferenceRequiredInputs: + """microcosm#1095: the required inputs are the ones the reference carries.""" + + _ENGINE_DEFAULTS = { + "employment_income": 0.0, + "dividend_income": 0.0, + "property_income": 0.0, + } + + def _frame(self, **columns): + return _person_frame( + { + "employment_income": np.asarray([0.0, 52_000.0]), + **{name: np.asarray(values) for name, values in columns.items()}, + } + ) + + def test_the_reference_supplies_the_required_inputs(self) -> None: + result = uk_release_input_coverage_gate( + self._frame(), + _StubEngine(self._ENGINE_DEFAULTS), + reference_surface=reference_surface("employment_income", "dividend_income"), + manifest=_manifest(()), + ) + assert not result.passed + assert result.details["missing"] == ["dividend_income"] + assert result.details["reference"] == reference_surface().label + assert result.details["required_from_reference"] == 2 + assert result.details["required_by_manifest"] == [] + + def test_a_reviewed_exclusion_beats_the_reference(self) -> None: + contract = _manifest( + ( + UKReleaseInputColumn( + "property_income", + "reviewed_exclusion", + reason="formula-owned in policyengine-uk", + tracking_note="microcosm#1106", + ), + ) + ) + result = uk_release_input_coverage_gate( + self._frame(), + _StubEngine(self._ENGINE_DEFAULTS), + reference_surface=reference_surface("employment_income", "property_income"), + manifest=contract, + ) + assert result.passed, result.failures + assert result.details["dormant_exclusions"] == ["property_income"] + assert result.details["required_from_reference"] == 1 + + def test_an_input_the_live_engine_does_not_load_is_not_required(self) -> None: + result = uk_release_input_coverage_gate( + self._frame(), + _StubEngine({"employment_income": 0.0}), + reference_surface=reference_surface("employment_income", "retired_input"), + manifest=_manifest(()), + ) + assert result.passed, result.failures + + def test_a_thin_reference_input_is_required_and_named_by_name_only(self) -> None: + result = uk_release_input_coverage_gate( + self._frame(dividend_income=[0.0, 500.0]), + _StubEngine(self._ENGINE_DEFAULTS), + reference_surface=reference_surface( + "employment_income", thin=("dividend_income",) + ), + manifest=_manifest(()), + ) + assert result.passed, result.failures + assert result.details["required_thin_in_reference"] == ["dividend_income"] + + def test_a_reference_measured_at_another_floor_is_refused(self) -> None: + with pytest.raises(ValueError, match="effective-mass floor"): + uk_release_input_coverage_gate( + self._frame(), + _StubEngine(self._ENGINE_DEFAULTS), + reference_surface=reference_surface("employment_income", floor=1e-3), + manifest=_manifest(()), + ) + + def test_a_vacuous_required_set_is_refused(self) -> None: + with pytest.raises(ValueError, match="vacuous"): + uk_release_input_coverage_gate( + self._frame(), + _StubEngine(self._ENGINE_DEFAULTS), + reference_surface=reference_surface(), + manifest=_manifest(()), + ) + + +def test_effective_input_signal_measures_by_the_gates_own_rule() -> None: + frame = _weighted_person_frame( + { + # Signal on positive mass: carried. + "employment_income": np.asarray([0.0, 52_000.0, 0.0]), + # Signal only on a zero-weight row: not carried. + "dividend_income": np.asarray([500.0, 0.0, 0.0]), + # Signal only on dust below the reviewed floor: not carried. + "savings_interest_income": np.asarray([0.0, 0.0, 10.0]), + # Every value the engine default: not carried. + "property_income": np.asarray([0.0, 0.0, 0.0]), + # Not a loader input: never measured. + "not_an_input": np.asarray([1.0, 1.0, 1.0]), + }, + np.asarray([0.0, 1_000.0, 1e-9]), + ) + engine = _StubEngine( + { + "employment_income": 0.0, + "dividend_income": 0.0, + "savings_interest_income": 0.0, + "property_income": 0.0, + } + ) + carried = uk_effective_input_signal( + frame, engine, minimum_nondefault_mass_share=1e-6 + ) + assert set(carried) == {"employment_income"} + entity, signal = carried["employment_income"] + assert entity == "person" + assert signal.tolist() == [False, True, False] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py index 45ce27f898..ece8f90559 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_input_coverage_manifest.py @@ -130,43 +130,32 @@ def test_hmrc_family_period_fields_come_from_the_bytes_their_hash_names() -> Non assert "canonical_source_manifest" not in family -def test_promoted_manifest_requires_the_full_reference_surface() -> None: - reference = _resource("efrs_parity_reference.json") +def test_manifest_declares_the_exclusions_and_leaves_the_rest_to_the_reference() -> ( + None +): + # microcosm#1095: the required inputs are the ones the cut's reference + # build carries, measured at gate time; the manifest declares only the + # reviewed exclusions: the engine-derived totals whose persisted + # components the reference carries (microcosm#1106, microcosm#1360) and + # the PIP categories the engine derives from the points (microcosm#1095). manifest = _resource("release_input_coverage_manifest.json") - assert manifest["counts"] == { - "required": 141, - "reviewed_exclusion": 4, - "total": 145, - } - assert set(manifest["columns"]) == set(reference["nonzero_shares"]) - # Every reference input is required except the engine-derived totals; their - # persisted components stay required (microcosm#1106, microcosm#1360). - excluded = { - name: entry - for name, entry in manifest["columns"].items() - if entry != {"status": "required"} - } - assert set(excluded) == { + assert manifest["counts"] == {"reviewed_exclusion": 4} + assert set(manifest["columns"]) == { "pip_dl_category", "pip_m_category", "property_wealth", "tax_free_savings_income", } - for name in excluded: - assert excluded[name]["status"] == "reviewed_exclusion" - for component in ( - "main_residence_value", - "other_residential_property_value", - "non_residential_property_value", - "savings_interest_income", - ): - assert manifest["columns"][component] == {"status": "required"} + for entry in manifest["columns"].values(): + assert entry["status"] == "reviewed_exclusion" + assert entry["reason"].strip() and entry["tracking_note"].strip() + assert manifest["reference"]["kind"] == "current_build" assert manifest["restoration_evidence"] == { "derived_from": "efrs_parity_known_gaps.json", "required_columns": ["charitable_investment_gifts", "gift_aid"], } assert manifest["candidate_evidence"]["tier"] == "frs" - assert manifest["schema_version"] == 3 + assert manifest["schema_version"] == 4 assert manifest["effective_mass_coverage"] == { "weight_source": "household_weight", "minimum_nondefault_mass_share": 1e-6, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index 8543b7d9bf..1cff5df858 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -1508,6 +1508,23 @@ def test_uk_dense_role_refuses_the_national_reference_flags(tmp_path) -> None: ) +def test_uk_national_role_refuses_the_dense_reference_surface(tmp_path) -> None: + # The national cut's certify step reads its reference surface + # (microcosm#1095); the national build runs no export-surface gate. + builder = candidate._load_builder_module() + (tmp_path / "spine.h5").write_bytes(b"spine") + parsed = builder._parse_args( + _argv( + tmp_path / "spine.h5", + tmp_path / "out", + "--reference-surface", + str(tmp_path / "reference_surface.json"), + ) + ) + with pytest.raises(ValueError, match="--reference-surface"): + builder._validate_cli_args(parsed) + + def test_uk_national_role_requires_the_reference_pair(monkeypatch, tmp_path): pytest.importorskip("tables") builder = candidate._load_builder_module() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py index 1fd76fb92f..d914a69e2a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py @@ -2,8 +2,10 @@ from __future__ import annotations +import json import math from datetime import UTC, date, datetime +from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -301,18 +303,57 @@ def test_ported_june_parity_gates_retain_their_named_failures() -> None: assert fit.name == "target_fit" -def test_export_surface_allows_claimant_roles_but_not_unreviewed_columns() -> None: +def test_export_surface_allows_declared_extras_but_not_unreviewed_columns() -> None: reference = {"person.age"} - assert uk_export_surface_gate( - reference | {"person.is_uc_claimant"}, reference - ).passed + assert uk_export_surface_gate(reference | {"household.lsoa_code"}, reference).passed unrelated = uk_export_surface_gate( - reference | {"person.is_uc_claimant", "person.unreviewed_extra"}, reference + reference | {"household.lsoa_code", "person.unreviewed_extra"}, reference ) assert not unrelated.passed assert any("unreviewed_extra" in failure for failure in unrelated.failures) +def test_export_surface_reports_a_declared_extra_the_reference_now_carries() -> None: + # microcosm#1095: a promotion moves the reference past a declared extra; + # the entry is reported as redundant, never failed. + reference = {"person.age", "household.lsoa_code"} + result = uk_export_surface_gate(reference, reference) + assert result.passed + assert result.details["redundant_allowed_extra_columns"] == ["household.lsoa_code"] + + +def test_export_surface_names_the_reference_build() -> None: + result = uk_export_surface_gate( + {"person.age"}, + {"person.age", "person.dropped_input"}, + reference_name="microcosm-uk-2024-25-national@build-a", + ) + assert not result.passed + assert "microcosm-uk-2024-25-national@build-a" in result.failures[0] + assert result.details["reference_name"] == "microcosm-uk-2024-25-national@build-a" + + +def test_export_surface_extras_are_not_exported_by_the_current_national_build() -> None: + # The standing list holds only what the current national build does not + # export (microcosm#1095): the dense line's geography and clone columns. + from microcosm.build.uk_runtime.terminal_gates import ( + UK_ALLOWED_EXTRA_EXPORT_COLUMNS, + ) + + gates = json.loads( + ( + Path(__file__).resolve().parents[3] / "src/microcosm/build/uk/gates.json" + ).read_text() + ) + entry = next(gate for gate in gates["gates"] if gate["id"] == "uk_export_surface") + assert entry["parameters"]["allowed_extra_columns"] == list( + UK_ALLOWED_EXTRA_EXPORT_COLUMNS + ) + assert len(UK_ALLOWED_EXTRA_EXPORT_COLUMNS) == len( + set(UK_ALLOWED_EXTRA_EXPORT_COLUMNS) + ) + + def test_export_candidate_columns_and_release_frame_carry_consumer_area_code_names() -> ( None ): @@ -398,7 +439,6 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: uk_release_export_frame, ) from microcosm.build.uk_runtime.terminal_gates import ( - UK_ALLOWED_EXTRA_EXPORT_COLUMNS, UK_REVIEWED_EXPORT_EXCLUSIONS, uk_export_candidate_columns, ) @@ -435,8 +475,9 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: } exported = uk_release_export_frame(frame) # microcosm#1106: the WAS total leaves; its engine components stay. - # microcosm#1194: the build identifiers stay beside the renumbered ones, - # as reviewed extras of the export surface. + # microcosm#1194: the build identifiers stay beside the renumbered ones; + # the current national build exports them too, so since microcosm#1095 + # they are part of the reference surface, not declared extras. carriers = { f"{entity}.{column}" for entity, column in UK_BUILD_IDENTIFIER_COLUMNS.items() } @@ -447,7 +488,6 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: "household.household_weight", *carriers, } - assert carriers <= set(UK_ALLOWED_EXTRA_EXPORT_COLUMNS) assert exported.weights_for("household").values.tolist() == [1.0, 2.0] assert exported.mass_log == frame.mass_log # The boundary drops the reviewed export exclusions, the internal diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_deduction_attributes.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_deduction_attributes.py index aa74492505..696a33f85d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_deduction_attributes.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_deduction_attributes.py @@ -137,8 +137,8 @@ def test_persisted_draws_are_not_the_engine_fallback_and_stay_exported() -> None that is pinned here alongside the divergence. """ - from microcosm.build.uk_runtime.terminal_gates import ( - UK_ALLOWED_EXTRA_EXPORT_COLUMNS, + from microcosm.build.uk_runtime.national_frame import ( + UK_RELEASE_EXPORT_DROPPED_COLUMNS, ) ids = np.arange(1, 4097, dtype=np.int64) @@ -148,5 +148,7 @@ def test_persisted_draws_are_not_the_engine_fallback_and_stay_exported() -> None assert agreement < 1e-3, agreement for column in ("uc_deduction_random_draw", "uc_deduction_type_random_draw"): - assert f"benunit.{column}" in UK_ALLOWED_EXTRA_EXPORT_COLUMNS + # The draws leave with the release (part of the current build's + # export surface since microcosm#1095, no longer a declared extra). + assert column not in UK_RELEASE_EXPORT_DROPPED_COLUMNS.get("benunit", ()) assert column in _stage().outputs diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index ff49f5593f..32dc7fc6df 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -567,13 +567,13 @@ def _is_hierarchy_diagnostics_version(value: object) -> bool: # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "ae9257b96da095b1c950f57727c0abb2c392e0cae8be23498f70eb3b6fb2a3f9" + "f40c27c3dbea19a5feaad8befb44eb5c5659f5c7324654baf5a057caec18b6c7" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "75d36c4b3001aa0c2d902da239e4f1a99f5693ad2b2083f74b82093ed0700126" + "ea6212e014fd04a706052ce61ea2bf38bc9f3d51a56dbd485a4ef53f3b07a9be" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "23bcfbf72d5e1f28b4d127cb79e8513b0c9b8de2c9bf2c61aab139a7e4475906" + "55ac83cf85793249124561579c179eef7ed9556d3781c76f011ccfff401cd0ee" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -1136,10 +1136,10 @@ def _uk_line_cut_tag_re(release_id: str) -> re.Pattern[str] | None: }, "release_cut": { "gates_manifest_sha256": ( - "c829049dcb1f52d3903985ecd71d01e21f09b1193bf11d47550f0fa47dc301e1" + "26c8c3acb20c314143019ccb4859c20eebe06d0b4b5961433ba2800d7c0b795e" ), "policy_sha256": ( - "76511c61eaf7a026165a2056c75dc79e525b0d12d017fbd902e5ecda2d95cd51" + "920d90b73e79523c72f95d86b2b0c7a99a5b7925e44fd559481991883f32ddec" ), }, } diff --git a/packages/microcosm-data/tests/engine_free/shared/test_contract.py b/packages/microcosm-data/tests/engine_free/shared/test_contract.py index 7087eda89e..ebce5f0ed5 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_contract.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_contract.py @@ -206,13 +206,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "ae9257b96da095b1c950f57727c0abb2c392e0cae8be23498f70eb3b6fb2a3f9" + "f40c27c3dbea19a5feaad8befb44eb5c5659f5c7324654baf5a057caec18b6c7" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "75d36c4b3001aa0c2d902da239e4f1a99f5693ad2b2083f74b82093ed0700126" + "ea6212e014fd04a706052ce61ea2bf38bc9f3d51a56dbd485a4ef53f3b07a9be" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "23bcfbf72d5e1f28b4d127cb79e8513b0c9b8de2c9bf2c61aab139a7e4475906" + "55ac83cf85793249124561579c179eef7ed9556d3781c76f011ccfff401cd0ee" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "f246388de42654dd68dbbb40c42282001b5b0a8bf48976162bcb1199085cb1d3" diff --git a/test_support/microcosm_build/uk_cut_reference.py b/test_support/microcosm_build/uk_cut_reference.py index c45eb56039..e19a816306 100644 --- a/test_support/microcosm_build/uk_cut_reference.py +++ b/test_support/microcosm_build/uk_cut_reference.py @@ -196,6 +196,7 @@ def stub_reference_evidence(reference, h5_path, output_dir): from microcosm.build.uk_runtime.cut_reference import ( UK_CUT_REFERENCE_INPUT_MASS_NAME, + UK_CUT_REFERENCE_SURFACE_NAME, ) destination = Path(output_dir) / UK_CUT_REFERENCE_INPUT_MASS_NAME @@ -217,4 +218,41 @@ def stub_reference_evidence(reference, h5_path, output_dir): } ) ) - return {"input_mass_reference": destination} + surface = Path(output_dir) / UK_CUT_REFERENCE_SURFACE_NAME + surface.write_text( + json.dumps( + { + "schema_version": 1, + "kind": "uk_reference_surface", + "label": reference.label, + "sha256": reference.sha256, + "minimum_nondefault_mass_share": 1e-6, + "export_columns": ["person.employment_income"], + "populated_inputs": ["employment_income"], + "thin_inputs": [], + } + ) + ) + return {"input_mass_reference": destination, "reference_surface": surface} + + +def reference_surface( + *carried: str, + thin: tuple[str, ...] = (), + export_columns: tuple[str, ...] = ("person.employment_income",), + label: str = "microcosm-uk-2024-25-national@uk-frs-calibration-attempt-test", + sha256: str = "0" * 64, + floor: float = 1e-6, +): + """A reference surface carrying ``carried`` inputs (and ``thin`` ones).""" + + from microcosm.build.uk_runtime.reference_surface import UKReferenceSurface + + return UKReferenceSurface( + label=label, + sha256=sha256, + export_columns=frozenset(export_columns), + populated_inputs=frozenset(carried), + thin_inputs=frozenset(thin), + minimum_nondefault_mass_share=floor, + ) diff --git a/test_support/microcosm_build/uk_release_input_coverage.py b/test_support/microcosm_build/uk_release_input_coverage.py index b27357aedf..ddcabc82e1 100644 --- a/test_support/microcosm_build/uk_release_input_coverage.py +++ b/test_support/microcosm_build/uk_release_input_coverage.py @@ -28,6 +28,7 @@ uk_release_input_coverage_gate, ) from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights +from test_support.microcosm_build.uk_cut_reference import reference_surface from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -121,6 +122,11 @@ def variable_entities(self, names) -> dict[str, str]: return {name: self._entities[name] for name in names if name in self._entities} +#: A reference that carries no input: the test's manifest declares the +#: required inputs itself, as every test did before microcosm#1095. +NO_REFERENCE_INPUTS = reference_surface(export_columns=("person.age",)) + + def _manifest( columns: tuple[UKReleaseInputColumn, ...], *, diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index a90a11c197..5d9f9f8b75 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -858,53 +858,42 @@ def build_manifest( ) -> dict[str, Any]: reference = reference or _load(REFERENCE_PATH) known_gaps_payload = known_gaps_payload or _load(KNOWN_GAPS_PATH) + # The ledger is still checked against the frozen enhanced-FRS facts it + # was measured on; neither decides the required inputs any more. _validate_known_gaps(reference, known_gaps_payload) known_gaps = known_gaps_payload["known_gaps"] restored_required = set(known_gaps_payload["restored_required_columns"]) - populated_layers = { - name - for name, share in reference["nonzero_shares"].items() - if float(share) > 0.0 - } - columns: dict[str, dict[str, str]] = {} engine_derived = known_gaps_payload["engine_derived_exclusions"] - for name in sorted(populated_layers): - exclusion = known_gaps.get(name) or engine_derived.get(name) - if exclusion is not None: - columns[name] = { - "status": "reviewed_exclusion", - "reason": str(exclusion["reason"]), - "tracking_note": str(exclusion["tracking_note"]), - } - else: - columns[name] = {"status": "required"} - - required = sorted( - name for name, entry in columns.items() if entry["status"] == "required" - ) - reviewed = sorted( - name - for name, entry in columns.items() - if entry["status"] == "reviewed_exclusion" - ) - source = reference["source"] + # Schema 4 (microcosm#1095): the required inputs are the ones the cut's + # reference build carries, measured at gate time, so the manifest + # declares only the reviewed exclusions. + columns: dict[str, dict[str, str]] = {} + for name in sorted({*known_gaps, *engine_derived}): + exclusion = known_gaps.get(name) or engine_derived[name] + columns[name] = { + "status": "reviewed_exclusion", + "reason": str(exclusion["reason"]), + "tracking_note": str(exclusion["tracking_note"]), + } candidate_source = known_gaps_payload["candidate_evidence"]["source"] return { - "schema_version": 3, + "schema_version": 4, "description": ( - "Declared full-coverage contract for a UK release: every populated " - "effective loader input in the pinned enhanced FRS must be persisted " - "with non-default signal, or carry a reviewed exclusion with the " - "campaign reason and tracking note." + "Release input-coverage contract for a UK cut: every loader input " + "the current build of the cut's line carries must be persisted " + "with non-default signal on the reviewed share of effective " + "population mass, unless it carries a reviewed exclusion here " + "with its reason and tracking note (microcosm#1095)." ), "reference": { - "derived_from": REFERENCE_PATH.name, - "filename": str(source["filename"]), - "revision": str(source["revision"]), - "sha256": str(source["sha256"]), - "vintage": str(source["vintage"]), - "period": str(source["period"]), - "populated_input_columns": len(populated_layers), + "kind": "current_build", + "required_inputs": ( + "The loader inputs the current build of the cut's line " + "carries, measured by the coverage gate's own rule; the " + "cut's reference step records them in reference_surface.json. " + "Until microcosm#1095 they were the populated inputs of the " + "frozen enhanced-FRS reference." + ), }, "candidate_evidence": { "derived_from": KNOWN_GAPS_PATH.name, @@ -1040,25 +1029,17 @@ def build_manifest( ), }, "derivation": ( - "Surface = efrs_parity_reference.json populated effective loader " - "inputs. status='required' when the sha-pinned candidate evidence " - "records non-default signal on at least the reviewed owning-entity " - "effective-mass share OR the column is pinned in " - "restored_required_columns after a source-family restoration; all " - "remaining surface columns are reviewed_exclusion with reason " + "Columns = the reviewed exclusions of efrs_parity_known_gaps.json: " + "its known gaps, with reason " f"{EXCLUSION_REASON!r} and a UK_COVERAGE_PROGRESS.md tracking note, " - "except the engine_derived_exclusions, which are reviewed_exclusion " - "with their own reason because the engine computes them from " - "persisted inputs. " - "The final release gate applies the same effective-mass floor, and " - "distributional restorations must pass it on their required source " - "channel." + "and its engine_derived_exclusions, with their own reason because " + "the engine computes them from persisted inputs. Every other input " + "the cut's reference build carries is required. The gate applies " + "one effective-mass floor to the reference and the candidate, and " + "distributional restorations must pass it on their required " + "source channel." ), - "counts": { - "required": len(required), - "reviewed_exclusion": len(reviewed), - "total": len(columns), - }, + "counts": {"reviewed_exclusion": len(columns)}, "columns": columns, } @@ -1394,8 +1375,9 @@ def main() -> int: _write_or_check(MANIFEST_PATH, manifest, check=args.check) action = "current" if args.check else "wrote" print( - f"{action}: {MANIFEST_PATH} — {manifest['counts']['required']} required, " - f"{manifest['counts']['reviewed_exclusion']} reviewed exclusions" + f"{action}: {MANIFEST_PATH} — " + f"{manifest['counts']['reviewed_exclusion']} reviewed exclusions; the " + "required inputs come from each cut's reference build" ) return 0 diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 27898626c4..52494d82d3 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -60,7 +60,7 @@ load_uk_calibration_measure_exclusions, ) from microcosm.build.uk_runtime.national_frame import load_uk_national_frame -from microcosm.build.uk_runtime.parity_reference import load_efrs_parity_reference +from microcosm.build.uk_runtime.reference_surface import load_uk_reference_surface from microcosm.build.uk_runtime.release_certification import ( compose_uk_release_certification, rehydrate_uk_fit_weight_records, @@ -333,11 +333,11 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: # (importance weights, mass receipts); the candidate is calibrated. spine_frame, _spine_provenance = load_uk_national_frame(args.spine_h5) engine = PolicyEngineUKCoverageEngine() + reference_surface = _reference_surface(args) parity_evidence = uk_release_parity_evidence( frame, diagnostics_targets=diagnostics["targets"], reference_registry=reference_registry, - parity_reference=load_efrs_parity_reference(), ) report = run_uk_release_cut_battery( frame, @@ -353,6 +353,7 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: parity_evidence=parity_evidence, fit_weight_records=rehydrate_uk_fit_weight_records(sidecar), input_mass_reference=_input_mass_reference(args), + reference_surface=reference_surface, exclusions_evaluated_on=evaluated_on, spine_frame=spine_frame, ) @@ -412,6 +413,25 @@ def _input_mass_reference(args: argparse.Namespace): return loaded +def _reference_surface(args: argparse.Namespace): + """The cut's reference surface, refused unless it is the cut's reference. + + The export-surface and input-coverage gates compare with the build the + score and the input-mass gate compare with (microcosm#1095). + """ + + reference = json.loads(args.reference_record.read_text()) + loaded = load_uk_reference_surface(args.reference_surface) + label = f"{reference.get('release')}@{reference.get('build')}" + if loaded.label != label or loaded.sha256 != reference.get("sha256"): + raise SystemExit( + f"error: the reference surface names {loaded.label} " + f"({loaded.sha256}), not the cut's reference {label}; write it from " + "the cut's reference (tools/run_uk_cut.py's reference step)." + ) + return loaded + + def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-h5", required=True, type=Path) @@ -430,6 +450,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--ledger-facts-sha256", required=True, type=_sha256) parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) parser.add_argument("--input-mass-reference", required=True, type=Path) + parser.add_argument( + "--reference-surface", + required=True, + type=Path, + help="The export surface and carried inputs of the cut's reference.", + ) parser.add_argument("--score-receipt", required=True, type=Path) parser.add_argument( "--reference-record", @@ -459,6 +485,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: "--certification-json": args.certification_json, "--score-receipt": args.score_receipt, "--reference-record": args.reference_record, + "--reference-surface": args.reference_surface, } resolved: dict[Path, str] = {} for flag, path in distinct.items(): diff --git a/tools/resolve_uk_reference.py b/tools/resolve_uk_reference.py index 3d8bbb6b6a..7cc289de68 100644 --- a/tools/resolve_uk_reference.py +++ b/tools/resolve_uk_reference.py @@ -15,9 +15,11 @@ the dataset, verifies its SHA-256, copies it there and writes ``reference.json`` beside it, with ``input_mass_reference.json`` (the build's input-mass totals and carriers, licensed-data aggregates that stay -with the run). A dense build takes ``--input-mass-reference`` from it. It -refuses a directory that already holds a reference, so a run keeps the one -it started with. +with the run) and ``reference_surface.json`` (the columns it exports and the +loader inputs it carries, names and flags only). A dense build takes +``--input-mass-reference`` and ``--reference-surface`` from it. It refuses a +directory that already holds a reference, so a run keeps the one it started +with. Reading ``policyengine/populace-uk-private`` needs ``HF_TOKEN``. """ @@ -60,8 +62,9 @@ def fetch_reference( """Download the current build of ``release`` into ``output_dir``. Beside the dataset and its record go the evidence the gates read from it: - its input-mass totals and carriers (``input_mass_reference.json``), which - a dense build takes with ``--input-mass-reference``. + its input-mass totals and carriers (``input_mass_reference.json``) and its + surface (``reference_surface.json``), which a dense build takes with + ``--input-mass-reference`` and ``--reference-surface``. """ reference = fetch_uk_cut_reference(release, output_dir, hub_download=hub_download) diff --git a/tools/run_uk_cut.py b/tools/run_uk_cut.py index 4105f016bd..4109ac26ad 100644 --- a/tools/run_uk_cut.py +++ b/tools/run_uk_cut.py @@ -65,6 +65,7 @@ from microcosm.build.uk_runtime.cut_reference import ( UK_CUT_REFERENCE_INPUT_MASS_NAME, UK_CUT_REFERENCE_RECORD_NAME, + UK_CUT_REFERENCE_SURFACE_NAME, UKCutReference, check_uk_cut_reference, fetch_uk_cut_reference, @@ -618,6 +619,10 @@ def reference_h5(self, reference: UKCutReference) -> Path: def reference_input_mass(self) -> Path: return self.reference_dir / UK_CUT_REFERENCE_INPUT_MASS_NAME + @property + def reference_surface(self) -> Path: + return self.reference_dir / UK_CUT_REFERENCE_SURFACE_NAME + @property def score_receipt(self) -> Path: return self.run_dir / "score_vs_reference.json" @@ -764,6 +769,7 @@ def certify_argv( "--ledger-facts-sha256", chronicle["consumer_facts.jsonl"], "--ledger-manifest-sha256", chronicle["manifest.json"], "--input-mass-reference", str(paths.reference_input_mass), + "--reference-surface", str(paths.reference_surface), "--score-receipt", str(paths.score_receipt), "--reference-record", str(paths.reference_record), "--release-id", RELEASE_ID, From c9c1bd720673fbcbc641fa1aacf216634ea75662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:36:14 +0000 Subject: [PATCH 06/18] Document UK cuts comparing with the current build Part of #1095. The operator docs described the enhanced-FRS reference, the incumbent flags and the rule-1 verdict. They now walk through the reference step, the files it writes, the --reference-* flags of the national role, the certifier and the dense build, and the score as a report with no verdict. - uk-cut-anyone.md: the reference step, its directory, and how to check or move what a cut compares with; the eFRS and input-mass input rows go. - Runbooks 806, 623 and 762: resolve the reference, then pass it to the build, the scorer, the certifier and the dense build. - uk-staging-operations.md, gate-battery-contract.md and the README: the score_vs_reference artifact, the reference_surface artifact and the current_build reference rule. - A changelog fragment for the change. Co-Authored-By: Claude Opus 5.5 --- ...5-certify-against-current-build.changed.md | 1 + docs/gate-battery-contract.md | 29 +++--- docs/uk-cut-anyone.md | 73 +++++++++------ docs/uk-dense-release-assembly-runbook-762.md | 29 ++++-- docs/uk-national-calibration-runbook-623.md | 74 +++++++++------ ...k-national-release-assembly-runbook-806.md | 89 +++++++++++++------ docs/uk-staging-operations.md | 15 ++-- packages/microcosm-build/README.md | 9 +- tools/run_uk_cut.py | 3 +- 9 files changed, 211 insertions(+), 111 deletions(-) create mode 100644 changelog.d/1095-certify-against-current-build.changed.md diff --git a/changelog.d/1095-certify-against-current-build.changed.md b/changelog.d/1095-certify-against-current-build.changed.md new file mode 100644 index 0000000000..c74b26c09d --- /dev/null +++ b/changelog.d/1095-certify-against-current-build.changed.md @@ -0,0 +1 @@ +UK cuts check and certify their candidate against the current build of their own line, as the Hub's `releases.json` names it, instead of the enhanced FRS (microcosm#1095, now that the migration in microcosm#665 is complete): national against national, dense against dense. A cut resolves its reference once, refusing a build that is withdrawn or not certified, and keeps its record, the dataset and the evidence the gates read from it with the run (`tools/run_uk_cut.py`'s new `reference` step, or `tools/resolve_uk_reference.py` outside it). The comparison of the candidate's target fit with the reference's is a report with no verdict (`score_vs_reference.json`), built by one country-neutral head-to-head module that the US comparison tool now shares. The input-mass gate compares each column's total with the reference's at the reference's own solved weights, within the largest relative drift measured between the last two national builds, and withholds the columns fewer than three source households carry. The export-surface gate compares with the reference's exported columns, so the declared extras shrink from 250 to the 26 the current national build does not export, and an entry the reference now carries is reported as redundant rather than failed. The input-coverage gate requires every loader input the reference carries, measured by the gate's own rule, less the coverage manifest's reviewed exclusions. The certification and the release contract move to schema 2 with a `reference` block, the dense build takes `--input-mass-reference` and `--reference-surface` from its own line's current build, and the frozen enhanced-FRS facts stay only as a historical record. diff --git a/docs/gate-battery-contract.md b/docs/gate-battery-contract.md index 10662fa269..b8ffb08f7d 100644 --- a/docs/gate-battery-contract.md +++ b/docs/gate-battery-contract.md @@ -126,7 +126,8 @@ Canonical artifact keys: | `coverage_engine`, `coverage_manifest` (optional override) | the UK national build | `release_input_coverage`, both phases | | `build_stage_names` | the UK national build preflight | `source_coverage` | | `parity_evidence` | the UK national build | `export_surface`, `target_surface`, `target_fit` | -| `input_mass_reference` | the UK national build, when the licensed sidecar is supplied | `input_mass_parity` (UK override) | +| `reference_surface` | the UK release cut (`--reference-surface`), and the dense build when given one | `export_surface` (the reference's exported columns), `release_input_coverage` (the inputs it carries), microcosm#1095 | +| `input_mass_reference` | the UK release cut and the dense build: the current build's totals and carriers, from the cut's reference step | `input_mass_parity` (UK override) | | `reviewed_input_mass_exclusions` (optional override) | the UK national build | `input_mass_parity` (UK override) | | `uk_area_support_summary`, `exclusions_evaluated_on` | the UK local rowwise build | `area_support` (UK binding; reviewed exclusions state the area's figures beside the floors it misses, and stale entries fail; they are gate-only and defer none of the area's cells, microcosm#1301) | | `local_target_diagnostics`, `exclusions_evaluated_on` | the UK local rowwise build | `per_family_fit` (the diagnostics only); `target_fit` on `surface: local_candidate` (UK binding; the local cells stand on `local_target_fit_reviewed_exclusions.json`, keyed `family/area_code/metric`, which lifts the fence only and is evaluated on the build's review date, microcosm#1301) | @@ -229,7 +230,7 @@ stretches on declaration alone: | --- | --- | --- | | phases | `terminal` | `preflight`, `assembled`, `transferred`, `terminal` | | entries | 9 | 73 (64 national-build entries, 7 local-candidate entries and 2 local diagnostics) | -| incumbent posture | none — incumbent-comparison gates deliberately not selected; external oracles replace self-parity | full — export/target parity against the pinned enhanced-FRS incumbent | +| incumbent posture | none — incumbent-comparison gates deliberately not selected; external oracles replace self-parity | full — export surface, input coverage and input mass against the current build of the cut's own line (microcosm#1095; until then the pinned enhanced-FRS incumbent) | | criticality mix | 8 blocking + 1 diagnostic | 71 blocking + 2 local diagnostics | The differences are entirely in the two country inputs — the spec file @@ -257,15 +258,21 @@ inputs — never implicit module state.** A country picks its reference on purpose and cannot silently inherit a mis-referenced comparison. The UK input-mass entry is the worked case. Its `parameters` declare the -active `reference`, the closed `reference_registry` of named identities plus -`totals_sha256` pins, the numeric thresholds, and the -`reviewed_exclusions_resource` for the schema-3 per-reference register. The -UK binding holds the runtime-supplied reference sidecar to the active named -descriptor, failing closed on identity or totals-digest drift; an exclusion -suppresses only for the active reference block. The export-surface entry declares its -reviewed comparison registers the same way (the allowed-extra column list -and the reviewed reference-side exclusions), with the hard-required, -never-waivable columns remaining a code-level guard noted in the entry. +active `reference` kind, `current_build` (microcosm#1095): the current build +of the cut's own line, which the cut's reference step resolves from the +Hub's `releases.json` and records, so the spec commits to the rule and the +cut's record to the build. They also declare the numeric thresholds and the +`reviewed_exclusions_resource` for the schema-3 per-reference register, whose +blocks are keyed by the reference's `@` label; an exclusion +suppresses only for the reference it names. The export-surface entry declares +its reviewed comparison registers the same way (the allowed-extra column +list, now only what the current national build does not export, and the +reviewed reference-side exclusions), with the hard-required, never-waivable +columns remaining a code-level guard noted in the entry. The input-coverage +entry requires what the reference carries, less the coverage manifest's +reviewed exclusions. Until microcosm#1095 all three compared with the pinned +enhanced FRS, through a closed registry of named identities with +`totals_sha256` pins. ## The onboarding checklist: every input a country supplies diff --git a/docs/uk-cut-anyone.md b/docs/uk-cut-anyone.md index 728b7f4e9f..a65438ba87 100644 --- a/docs/uk-cut-anyone.md +++ b/docs/uk-cut-anyone.md @@ -48,14 +48,18 @@ with `agent-secret get microcosm/uk-gate-signing-key-team maxghenis`, and 1. `fetch` 2. `preflight` -3. spine -4. national dry run, then the national calibration -5. re-score, only when the in-build evaluation did not pass -6. certify -7. stage-1 comparison (with `--previous-h5`) -8. assemble -9. publish preflight -10. publish (with `--publish`) +3. `reference`: the current national build the cut compares with + ([The reference build](#the-reference-build)) +4. spine +5. national dry run, then the national calibration +6. re-score, only when the build's comparison with the reference is missing + or names another candidate or reference +7. certify +8. stage-1 comparison, against the reference (`--previous-h5` names another + file) +9. assemble +10. publish preflight +11. publish (with `--publish`) Every step first checks whether its outputs already exist and verify, and skips itself if they do. So when a run stops, fix the cause and run the same @@ -85,6 +89,7 @@ derive from licensed data: | Path | Contents | |---|---| | `inputs/` | shared verified inputs | +| `cuts/