From 5266d9563a8d65665c091259f3a5e0785dc71364 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 14 Oct 2026 15:39:17 +0100 Subject: [PATCH 1/7] Package UK area lists and certify UK regional releases Ship the 650 constituencies (2024) and 361 local authorities (April 2023, with Ards and North Down and real hexes for the 2023 unitaries) with the package. Area regions and geography output labels read them instead of GCS; the opt-in registry previously came back empty because its GCS helper no longer exists in policyengine-core. Extend --regional-manifest-uri to the UK: a Microcosm local-area release routes constituency and local-authority regions to its file, and --regional-release-kind area_coded_national routes them to the enhanced FRS with its weight matrices at their own revision, for a national-first switch. Certification records regional_release_kind, checks the regional release's own model compatibility, HEAD-checks carried and Microcosm UK artifacts, refuses re-certification that would drop regional routing, and names a build by its cut tag. Area regions take the routed dataset and join the default registry only for a local-area release. Add the Universal Credit coverage floor check from #491. Refs #553 Co-Authored-By: Claude Opus 5.5 --- changelog.d/553.added.md | 1 + changelog.d/553.fixed.md | 1 + docs/engineering/skills/data-certification.md | 92 ++- docs/regions.md | 60 +- docs/release-bundles.md | 7 + scripts/bundle.py | 23 +- scripts/certify_data_release.py | 31 +- scripts/check_uk_uc_floor.py | 57 ++ src/policyengine/countries/uk/regions.py | 131 ++-- .../data/qualification/uk_uc_floor.json | 32 + src/policyengine/data/uk_geography/SOURCES.md | 38 + .../data/uk_geography/constituencies_2024.csv | 651 ++++++++++++++++++ .../uk_geography/local_authorities_2023.csv | 362 ++++++++++ src/policyengine/data/uk_geography_assets.py | 24 + .../outputs/constituency_impact.py | 11 +- .../outputs/local_authority_impact.py | 12 +- .../outputs/uk_geography_impact.py | 14 +- src/policyengine/provenance/certification.py | 470 ++++++++++--- src/policyengine/provenance/manifest.py | 37 +- src/policyengine/provenance/uk_uc_floor.py | 159 +++++ tests/test_certify_data_release.py | 495 ++++++++++++- tests/test_constituency_impact.py | 31 +- tests/test_local_authority_impact.py | 37 +- tests/test_release_manifests.py | 35 + tests/test_uk_area_lists.py | 114 +++ tests/test_uk_regions.py | 141 ++++ tests/test_uk_uc_floor.py | 84 +++ 27 files changed, 2924 insertions(+), 226 deletions(-) create mode 100644 changelog.d/553.added.md create mode 100644 changelog.d/553.fixed.md create mode 100644 scripts/check_uk_uc_floor.py create mode 100644 src/policyengine/data/qualification/uk_uc_floor.json create mode 100644 src/policyengine/data/uk_geography/SOURCES.md create mode 100644 src/policyengine/data/uk_geography/constituencies_2024.csv create mode 100644 src/policyengine/data/uk_geography/local_authorities_2023.csv create mode 100644 src/policyengine/provenance/uk_uc_floor.py create mode 100644 tests/test_uk_area_lists.py create mode 100644 tests/test_uk_uc_floor.py diff --git a/changelog.d/553.added.md b/changelog.d/553.added.md new file mode 100644 index 000000000..d2e90df76 --- /dev/null +++ b/changelog.d/553.added.md @@ -0,0 +1 @@ +Ship the UK constituency (2024, 650 including Northern Ireland) and local-authority (April 2023, 361 including Ards and North Down) lists with the package, so area regions and geography output labels no longer depend on GCS. Certify a UK regional release with `certify-data --regional-manifest-uri`: a Microcosm local-area release, or, for a national-first switch, the enhanced FRS with its area weight matrices (`--regional-release-kind area_coded_national`). Constituency and local-authority regions load the routed dataset and join the default registry when it is a local-area release. Add the Universal Credit coverage floor check (`scripts/check_uk_uc_floor.py`) for UK default switches. diff --git a/changelog.d/553.fixed.md b/changelog.d/553.fixed.md new file mode 100644 index 000000000..64cac1129 --- /dev/null +++ b/changelog.d/553.fixed.md @@ -0,0 +1 @@ +Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose). diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 5f1d9abe0..cbd06576d 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -23,9 +23,75 @@ the checkout is dirty). ```bash python scripts/bundle.py certify-data --country uk --data-producer populace \ - --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" ``` +Pin the cut tag (`@`), never `main`: each cut of a Microcosm release +line rewrites that line's `releases//` directory, for example +`releases/microcosm-uk-2024-25-national/`. Microcosm records the line name as +`build.build_id` on every cut and the cut in `build.cut_tag`; certification +uses the cut tag as the build id (`data_releases..version`, the +certification's `data_build_id` and the changelog fragment name), so two cuts +of one line never look identical. + +### UK constituency and local-authority regions + +UK constituency and local-authority regions row-filter the dataset a regional +release routes them to (`region_datasets.constituency` and +`region_datasets.local_authority`), and the national dataset when nothing is +routed. policyengine-sim-api accepts a UK default without area codes +(`constituency_code_oa`, `la_code_oa`) only when both are routed. Certify a +Microcosm national release with one of: + +- **A Microcosm local-area release (ship together).** The release declares + `dataset_role: non_default_local_area`, as for the US below. + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" \ + --model-version "" + ``` + + This records `regional_release_kind: local_area`, and the default UK region + registry then includes all 650 constituencies and 361 local authorities. + +- **The enhanced FRS (national first).** When the national release is ready + before a local one, route the area regions to a certified policyengine-uk-data + release whose households carry area codes: + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "hf://model/policyengine/policyengine-uk-data-private@/releases//release_manifest.json" \ + --regional-release-kind area_coded_national \ + --model-version "" + ``` + + Certification carries that release's national dataset + (`enhanced_frs_2024_25`) and its `weights` artifacts (the constituency and + local-authority matrices and `local_geography_weights`) at their own repo + and revision, records `regional_release_kind: area_coded_national`, and + leaves area regions out of the default registry: the API keeps serving them + from the matrices. The enhanced FRS release must itself claim the pinned + policyengine-uk version (built with it or listed in + `compatible_model_packages`); otherwise policyengine-uk-data must publish a + claim first. Its `frs_2024_25` is not carried, so keep it loadable with a + `dataset_overlays.uk` entry. + +Either way, certification checks the regional release's own model +compatibility and HEAD-checks every artifact it carries. + +### Re-certifying a routed bundle + +Certification rewrites `data_releases.` from the manifests it is +given. While the bundle routes regions (`regional_source_manifest_uri` is set), +a run without `--regional-manifest-uri` is refused; pass the current regional +release again (or its replacement), or `--drop-regional-release` to remove the +routing on purpose. + +### US + For US Populace certification, certify the Populace release manifest directly: ```bash @@ -84,8 +150,10 @@ Hard failures (certification refuses): missing national default dataset, default dataset absent from artifacts, any artifact without a revision pin, unreachable certified dataset, missing required supplemental release files (for example Populace-US `us_source_coverage.json`), a malformed shared -local-area release or malformed legacy per-state artifacts when -`--regional-manifest-uri` is used, unknown country. +local-area release, area-coded national release or legacy per-state artifacts +when `--regional-manifest-uri` is used, a regional release that does not serve +the pinned model version, an unreachable regional artifact, a re-certification +that would silently drop regional routing, unknown country. Certification gate: the model version must either exactly match the build-time model (`compatibility_basis: built_with_model_package`) or be @@ -98,6 +166,24 @@ refused: a new data build or a published compatibility claim is required. Warnings (recorded, not blocking): artifacts without sha256, and the publisher-claim basis above. +## Acceptance evidence for a UK default switch + +A PR that changes the UK default dataset carries, besides the UK candidate +qualification gates, the Universal Credit coverage floor from #491: benefit +units with Universal Credit and Universal Credit spend in Great Britain within +10% of DWP's and the OBR's figures. + +```bash +python scripts/check_uk_uc_floor.py --dataset \ + --year 2025 --output uc_floor.json +``` + +The benchmarks and their sources are in +`src/policyengine/data/qualification/uk_uc_floor.json`. The receipt holds +weighted national totals only. Before quoting any other figure from UK data +in a PR, apply the UKDS output rules: no unit-record values, no cells under +10 unweighted cases, no differencing. + ## Release runbooks Concrete, fill-in-the-id runbooks that replay a specific certification live diff --git a/docs/regions.md b/docs/regions.md index a6f3e2373..242c9bed8 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -86,6 +86,38 @@ england = pe.uk.model.get_region("country/england") Pass the registry entry's `scoping_strategy` to `Simulation` when using a national UK dataset. +## UK constituency and local-authority regions + +The package ships the area lists: the 650 parliamentary constituencies (2024 +boundaries, including Northern Ireland) and the 361 local authority districts +(April 2023, including Ards and North Down, `N09000011`), each with a name and +a hex-map position (`src/policyengine/data/uk_geography/`, sources in its +`SOURCES.md`). Building these regions needs no network access. + +Constituency regions (`constituency/`) row-filter `constituency_code_oa` +and local-authority regions (`local_authority/`) row-filter `la_code_oa`. +Their `dataset_path` is the dataset the certified bundle routes them to +(`data_releases.uk.region_datasets`), or `None`, meaning the national dataset, +when nothing is routed: + +| `regional_release_kind` | Area regions load | In the default registry | +| --- | --- | --- | +| none (today) | the national dataset | no | +| `area_coded_national` | another release's area-coded national dataset (the enhanced FRS), while the default is a national file without area codes | no | +| `local_area` | a certified local-area release | yes | + +Pass `include_constituencies=True` and `include_local_authorities=True` to +`build_uk_region_registry` to include them regardless. + +```python +from policyengine.countries.uk.regions import build_uk_region_registry + +registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True +) +registry.get("local_authority/N09000011").label # "Ards and North Down" +``` + ## UK parliamentary constituencies To publish results by constituency, local authority or region, use @@ -94,10 +126,9 @@ The functions in this section and the next are deprecated: their output is not disclosure-controlled. Constituency-level impacts group household output rows by the longwise -`constituency_code_oa` column carried by the dataset. If the constituency CSV is -available locally or from the PolicyEngine UK GCS bucket, PolicyEngine uses it -to attach names and map coordinates; otherwise results still compute and use -the code as the label. +`constituency_code_oa` column carried by the dataset. Names and map coordinates +come from the packaged constituency list; codes it does not list use the code +as the label. ```python from policyengine.outputs import compute_uk_constituency_impacts @@ -109,10 +140,9 @@ impacts = compute_uk_constituency_impacts( impacts.constituency_results ``` -To force a specific metadata file, pass `constituency_csv_path`. To avoid -downloading metadata and fall back to code-only labels, pass -`download_missing_assets=False`. The legacy `weight_matrix_path` and `year` -arguments are accepted for backward compatibility but ignored. +To use a different metadata file, pass `constituency_csv_path`. The legacy +`weight_matrix_path`, `year` and `download_missing_assets` arguments are +accepted for backward compatibility but ignored. ## UK local authorities @@ -126,11 +156,15 @@ impacts = compute_uk_local_authority_impacts( impacts.local_authority_results ``` -Local-authority impacts follow the same longwise pattern using `la_code_oa`. -Pass `local_authority_csv_path` to use a specific metadata CSV, or -`download_missing_assets=False` to skip metadata download and use code-only -labels. The legacy `weight_matrix_path` and `year` arguments are accepted for -backward compatibility but ignored. +Local-authority impacts follow the same longwise pattern using `la_code_oa`, +labelled from the packaged April 2023 list. Pass `local_authority_csv_path` to +use a specific metadata CSV. The legacy `weight_matrix_path`, `year` and +`download_missing_assets` arguments are accepted for backward compatibility +but ignored. + +Both impacts need simulations on a dataset whose households carry the area +codes. When the national default does not, run them on the dataset the bundle +routes the area regions to. ## Region registries diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 80b6f8500..632846eef 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -248,6 +248,13 @@ path. The national `default_dataset` remains unchanged. Ordinary bundle installation downloads only that national default; a regional runtime is responsible for materializing the additional certified dataset. +`data_releases.{country}.regional_release_kind` records how certification +routed those region types: `local_area` for a non-default local-area release, +and `area_coded_national` (UK only) for another release's national dataset +whose households carry area codes, certified together with that release's +area weight matrices at their own repo and revision. It is absent when nothing +is routed, or for a legacy US per-state release. + Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's primary `data_package`. This prevents runtime code from guessing how a diff --git a/scripts/bundle.py b/scripts/bundle.py index 072b2cb39..3efb69077 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -44,6 +44,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-artifact-prefix", args.regional_artifact_prefix]) if args.regional_path_template: argv.extend(["--regional-path-template", args.regional_path_template]) + if args.regional_release_kind: + argv.extend(["--regional-release-kind", args.regional_release_kind]) + if args.drop_regional_release: + argv.append("--drop-regional-release") if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -200,10 +204,25 @@ def _parser() -> argparse.ArgumentParser: certify.add_argument( "--regional-manifest-uri", help=( - "Optional regional release_manifest.json URI to merge into US " - "Populace certification." + "Regional release_manifest.json URI whose dataset serves the local " + "region types (US states and districts, UK constituencies and local " + "authorities)." ), ) + certify.add_argument( + "--regional-release-kind", + choices=["local_area", "area_coded_national"], + help=( + "How to read the regional release. Defaults to local_area; " + "area_coded_national (UK only) routes area regions to another " + "national release's area-coded dataset and weight matrices." + ), + ) + certify.add_argument( + "--drop-regional-release", + action="store_true", + help="Remove the bundle's current regional routing on re-certification.", + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 7a55ca2f9..ad7cef764 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -8,7 +8,11 @@ Usage:: python scripts/bundle.py certify-data --country uk --data-producer populace \\ - --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + +Pin the cut tag, never a branch: each cut rewrites ``releases//``. +Add ``--regional-manifest-uri`` to route local regions to a regional release +(see docs/engineering/skills/data-certification.md). Private data (UK) requires ``HUGGING_FACE_TOKEN`` or ``HF_TOKEN`` in the environment. After running: commit the changed bundle manifest, pyproject.toml, @@ -53,10 +57,29 @@ def main(argv=None) -> int: "--regional-manifest-uri", default=None, help=( - "Optional regional data release manifest to merge into US Populace " - "certification." + "Regional release_manifest.json URI whose dataset serves the local " + "region types (US states and districts, UK constituencies and local " + "authorities). Required on re-certification while the bundle routes " + "regions, unless --drop-regional-release is given." + ), + ) + parser.add_argument( + "--regional-release-kind", + default="local_area", + choices=["local_area", "area_coded_national"], + help=( + "local_area: a non-default local-area release. " + "area_coded_national (UK only): another certified national release " + "whose households carry area codes, such as the enhanced FRS; its " + "national dataset and area weight matrices serve constituency and " + "local-authority regions." ), ) + parser.add_argument( + "--drop-regional-release", + action="store_true", + help="Remove the bundle's current regional routing on re-certification.", + ) parser.add_argument( "--regional-artifact-prefix", default="states/", @@ -118,6 +141,8 @@ def main(argv=None) -> int: regional_manifest_uri=args.regional_manifest_uri, regional_artifact_prefix=args.regional_artifact_prefix, regional_path_template=args.regional_path_template, + regional_release_kind=args.regional_release_kind, + drop_regional_release=args.drop_regional_release, core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, ) diff --git a/scripts/check_uk_uc_floor.py b/scripts/check_uk_uc_floor.py new file mode 100644 index 000000000..d79c62719 --- /dev/null +++ b/scripts/check_uk_uc_floor.py @@ -0,0 +1,57 @@ +"""Check a UK dataset against the Universal Credit coverage floor. + +Usage:: + + python scripts/check_uk_uc_floor.py --dataset \\ + [--year 2025] [--output receipt.json] + +A bundle dataset name (for example ``enhanced_frs_2024_25``) is materialized +from the certified bundle; private UK data needs ``HUGGING_FACE_TOKEN``. Exits +non-zero when either measure is outside the tolerance. The receipt holds +weighted national totals only, so it can be attached to a certification PR. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from policyengine.provenance.uk_uc_floor import check_uc_floor # noqa: E402 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--dataset", + required=True, + help="A bundle dataset name or a local UK dataset .h5 path.", + ) + parser.add_argument("--year", type=int, default=2025) + parser.add_argument("--output", type=Path, help="Write the JSON receipt here.") + args = parser.parse_args(argv) + + if Path(args.dataset).expanduser().is_file(): + dataset_path = str(Path(args.dataset).expanduser()) + dataset = Path(dataset_path).name + else: + from policyengine.provenance.dataset_materialization import ( + materialize_dataset, + ) + + source = materialize_dataset("uk", args.dataset) + dataset_path, dataset = source.path, source.source_uri + + receipt = check_uc_floor(dataset_path, args.year, dataset=dataset) + text = receipt.model_dump_json(indent=2) + if args.output: + args.output.write_text(text + "\n") + print(text) + return 0 if receipt.passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/policyengine/countries/uk/regions.py b/src/policyengine/countries/uk/regions.py index f74dd1f2e..4e7a59cb7 100644 --- a/src/policyengine/countries/uk/regions.py +++ b/src/policyengine/countries/uk/regions.py @@ -3,25 +3,31 @@ This module defines all UK geographic regions: - National (1) - Countries (4: England, Scotland, Wales, Northern Ireland) -- Constituencies (loaded from CSV at runtime) -- Local Authorities (loaded from CSV at runtime) - -Note: Constituencies and local authorities filter from the national dataset -using geography columns carried on each household. This keeps subnational -scoping tied to the dataset rows, not to a separate weight matrix whose -household dimension can drift from the default dataset. +- Constituencies (650, from the packaged 2024 list) +- Local Authorities (361, from the packaged April 2023 list) + +Note: Constituencies and local authorities filter household rows on the +geography columns each household carries (``constituency_code_oa`` and +``la_code_oa``). This keeps subnational scoping tied to the dataset rows, not +to a separate weight matrix whose household dimension can drift from the +default dataset. They load the dataset the certified bundle routes them to +(``region_datasets``), and the national dataset when it routes none. + +Their codes, names and hex positions come from the area lists shipped with the +package (``data/uk_geography``), so building them needs no network access. """ +import csv import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from policyengine.core.region import Region, RegionRegistry from policyengine.core.scoping_strategy import RowFilterStrategy from policyengine.data.uk_geography_assets import ( - CONSTITUENCY_ASSET_SPEC, - LOCAL_AUTHORITY_ASSET_SPEC, UK_GEOGRAPHY_BUCKET_URI, + packaged_uk_area_list_path, ) +from policyengine.provenance import manifest as release_manifest from policyengine.provenance.manifest import resolve_region_dataset_path if TYPE_CHECKING: @@ -55,81 +61,58 @@ ) +def _load_area_list(geography_type: str) -> list[dict]: + with open(packaged_uk_area_list_path(geography_type), encoding="utf-8") as file: + return [ + {"code": row["code"], "name": row["name"]} for row in csv.DictReader(file) + ] + + def _load_constituencies_from_csv() -> list[dict]: - """Load UK constituency data from CSV. + """Load the 650 UK parliamentary constituencies (2024 boundaries). - Constituencies are loaded from: - gs://policyengine-uk-data-private/constituencies_2024.csv + Read from the packaged ``data/uk_geography/constituencies_2024.csv``. Returns: List of dicts with 'code' and 'name' keys """ - try: - from policyengine_core.tools.google_cloud import download - except ImportError: - # If policyengine_core is not available, return empty list - return [] - - try: - csv_path = download( - gcs_bucket=CONSTITUENCY_ASSET_SPEC.bucket, - gcs_key=CONSTITUENCY_ASSET_SPEC.lookup_csv_filename, - ) - import pandas as pd - - df = pd.read_csv(csv_path) - return [{"code": row["code"], "name": row["name"]} for _, row in df.iterrows()] - except (OSError, KeyError, ValueError) as exc: - logger.warning("Failed to load constituencies CSV: %s", exc) - return [] - except Exception: - logger.error("Unexpected error loading constituencies CSV", exc_info=True) - return [] + return _load_area_list("constituency") def _load_local_authorities_from_csv() -> list[dict]: - """Load UK local authority data from CSV. + """Load the 361 UK local authority districts (April 2023). - Local authorities are loaded from: - gs://policyengine-uk-data-private/local_authorities_2021.csv + Read from the packaged ``data/uk_geography/local_authorities_2023.csv``. Returns: List of dicts with 'code' and 'name' keys """ - try: - from policyengine_core.tools.google_cloud import download - except ImportError: - # If policyengine_core is not available, return empty list - return [] - - try: - csv_path = download( - gcs_bucket=LOCAL_AUTHORITY_ASSET_SPEC.bucket, - gcs_key=LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename, - ) - import pandas as pd + return _load_area_list("local_authority") - df = pd.read_csv(csv_path) - return [{"code": row["code"], "name": row["name"]} for _, row in df.iterrows()] - except (OSError, KeyError, ValueError) as exc: - logger.warning("Failed to load local authorities CSV: %s", exc) - return [] - except Exception: - logger.error("Unexpected error loading local authorities CSV", exc_info=True) - return [] + +def area_regions_routed_to_local_release() -> bool: + """Whether the bundle routes UK area regions to a local-area release. + + Constituency and local-authority regions join the default registry only + then. A national-first bundle routes them to an area-coded national + release instead, whose consumers keep their own area handling. + """ + manifest = release_manifest.get_release_manifest("uk") + return manifest.regional_release_kind == "local_area" def build_uk_region_registry( - include_constituencies: bool = False, - include_local_authorities: bool = False, + include_constituencies: Optional[bool] = None, + include_local_authorities: Optional[bool] = None, ) -> RegionRegistry: """Build the UK region registry. Args: - include_constituencies: If True, load and include constituencies from CSV. - Defaults to False to avoid GCS dependency at import time. - include_local_authorities: If True, load and include local authorities from CSV. - Defaults to False to avoid GCS dependency at import time. + include_constituencies: Whether to include the 650 constituencies. + ``None`` includes them only when the bundle routes area regions + to a local-area release. + include_local_authorities: Whether to include the 361 local + authorities. ``None`` follows the same rule. Returns: RegionRegistry containing: @@ -138,6 +121,13 @@ def build_uk_region_registry( - Optionally: constituencies (if include_constituencies=True) - Optionally: local authorities (if include_local_authorities=True) """ + if include_constituencies is None or include_local_authorities is None: + routed_to_local = area_regions_routed_to_local_release() + if include_constituencies is None: + include_constituencies = routed_to_local + if include_local_authorities is None: + include_local_authorities = routed_to_local + regions: list[Region] = [] # 1. National region (has dedicated dataset) @@ -166,9 +156,10 @@ def build_uk_region_registry( ) ) - # 3. Constituencies (optional, loaded from CSV) + # 3. Constituencies (packaged list) if include_constituencies: constituencies = _load_constituencies_from_csv() + constituency_dataset_path = resolve_region_dataset_path("uk", "constituency") for const in constituencies: regions.append( Region( @@ -176,6 +167,7 @@ def build_uk_region_registry( label=const["name"], region_type="constituency", parent_code="uk", + dataset_path=constituency_dataset_path, scoping_strategy=RowFilterStrategy( variable_name="constituency_code_oa", variable_value=const["code"], @@ -183,9 +175,12 @@ def build_uk_region_registry( ) ) - # 4. Local Authorities (optional, loaded from CSV) + # 4. Local authorities (packaged list) if include_local_authorities: local_authorities = _load_local_authorities_from_csv() + local_authority_dataset_path = resolve_region_dataset_path( + "uk", "local_authority" + ) for la in local_authorities: regions.append( Region( @@ -193,6 +188,7 @@ def build_uk_region_registry( label=la["name"], region_type="local_authority", parent_code="uk", + dataset_path=local_authority_dataset_path, scoping_strategy=RowFilterStrategy( variable_name="la_code_oa", variable_value=la["code"], @@ -203,7 +199,8 @@ def build_uk_region_registry( return RegionRegistry(country_id="uk", regions=regions) -# Default registry with just core regions (national + countries) -# To get full registry with constituencies/LAs, call: +# Default registry: national and countries, plus constituencies and local +# authorities when the bundle routes them to a local-area release. To include +# them regardless, call: # build_uk_region_registry(include_constituencies=True, include_local_authorities=True) uk_region_registry = build_uk_region_registry() diff --git a/src/policyengine/data/qualification/uk_uc_floor.json b/src/policyengine/data/qualification/uk_uc_floor.json new file mode 100644 index 000000000..53821225f --- /dev/null +++ b/src/policyengine/data/qualification/uk_uc_floor.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "description": "Universal Credit coverage floor for UK certification candidates (PolicyEngine/policyengine.py#491, #553). Benefit units with Universal Credit and Universal Credit spend in Great Britain must each be within the tolerance of DWP's and the OBR's figures for the same year. These are the totals policyengine-uk-data calibrates to (targets dwp/uc/households and obr/universal_credit).", + "tolerance": 0.1, + "geography": "great_britain", + "years": { + "2025": { + "benefit_units": { + "value": 6700000, + "source": "DWP, Universal Credit deductions statistics, March 2025 to February 2026, Table 1: households on Universal Credit (households with a deduction divided by their share), calendar-2025 average", + "url": "https://www.gov.uk/government/statistics/universal-credit-quarterly-statistics-29-april-2013-to-12-february-2026/universal-credit-deductions-statistics-march-2025-to-february-2026" + }, + "spend": { + "value": 79286349140, + "source": "OBR, Economic and fiscal outlook, March 2026, detailed forecast tables (expenditure), Table 4.9: Universal Credit inside and outside the welfare cap, 2025-26 (£66.411bn + £12.876bn; DWP social security, Great Britain)", + "url": "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/" + } + }, + "2026": { + "benefit_units": { + "value": 7200000, + "source": "DWP, Universal Credit deductions statistics, March 2025 to February 2026, Table 1: households on Universal Credit, December 2025 to February 2026 plateau", + "url": "https://www.gov.uk/government/statistics/universal-credit-quarterly-statistics-29-april-2013-to-12-february-2026/universal-credit-deductions-statistics-march-2025-to-february-2026" + }, + "spend": { + "value": 87649386699, + "source": "OBR, Economic and fiscal outlook, March 2026, detailed forecast tables (expenditure), Table 4.9: Universal Credit inside and outside the welfare cap, 2026-27 (£71.710bn + £15.939bn; DWP social security, Great Britain)", + "url": "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/" + } + } + } +} diff --git a/src/policyengine/data/uk_geography/SOURCES.md b/src/policyengine/data/uk_geography/SOURCES.md new file mode 100644 index 000000000..1e736684a --- /dev/null +++ b/src/policyengine/data/uk_geography/SOURCES.md @@ -0,0 +1,38 @@ +# UK area lists + +Codes, names and hex-map positions for the UK area regions and the labels of +geography outputs. They are not the lookup CSVs paired with the legacy +weight matrices, whose row order indexes those matrices. + +## `local_authorities_2023.csv` + +Columns `code,name,x,y`; 361 local authority districts on the April 2023 frame: +296 in England, 22 in Wales, 32 in Scotland and 11 in Northern Ireland, +including Ards and North Down (`N09000011`) and the four unitaries created on +1 April 2023 (`E06000063`–`E06000066`), and none of the 17 districts they +replaced. + +- Codes and names: ONS, Local Authority Districts (April 2023) Names and Codes + in the United Kingdom (columns `LAD23CD`, `LAD23NM`), ONS Open Geography + Portal item `e8b361ba9e98418ba8ff2f892d00c352`. Contains public sector + information licensed under the Open Government Licence v3.0. +- Hex positions: PolicyEngine's April 2023 local-authority hex layout, from + PolicyEngine/policyengine-sim-api + `projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/` + at commit `3ac09cf9205bc66a5809e70a634c457362d1a887`: + - `names.csv` sha256 `9e183902e54383d56f7f311d0eaac6a9ee92522a5f21c58ce5118265ae3f1bf4` + - `coordinates_lad23.csv` sha256 `b1489a5b0101ff6d91ef9262e992cca19b59359edb42d012c305a21a04571f64` +- Built by joining `coordinates_lad23.csv` to `names.csv` on `code`, sorted by + code. The names match the ONS file exactly. Against policyengine-uk-data's + `local_authorities_2021.csv` (360 rows on the same frame), this adds + `N09000011` and replaces the shared (0, 0) placeholder of the four 2023 + unitaries with their own hexes. + +## `constituencies_2024.csv` + +Columns `code,name,x,y,region,country`; the 650 parliamentary constituencies +of the 2024 boundaries: 543 in England, 57 in Scotland, 32 in Wales and 18 in +Northern Ireland. Copied unchanged from PolicyEngine/policyengine-uk-data +`policyengine_uk_data/storage/constituencies_2024.csv` at commit +`26ccdd1fd1aff6665704fda37388a86adecb5122` (sha256 `f4f35d61ffc004190c438a7e1583caefd85054f962671a00069aedd0a1b5b737`). +Codes and names: ONS (Open Government Licence v3.0). diff --git a/src/policyengine/data/uk_geography/constituencies_2024.csv b/src/policyengine/data/uk_geography/constituencies_2024.csv new file mode 100644 index 000000000..d0164c4a2 --- /dev/null +++ b/src/policyengine/data/uk_geography/constituencies_2024.csv @@ -0,0 +1,651 @@ +code,name,x,y,region,country +E14001063,Aldershot,56,-40,South East,England +E14001064,Aldridge-Brownhills,56,-30,West Midlands,England +E14001065,Altrincham and Sale West,52,-25,North West,England +E14001066,Amber Valley,58,-27,East Midlands,England +E14001067,Arundel and South Downs,61,-44,South East,England +E14001068,Ashfield,60,-27,East Midlands,England +E14001069,Ashford,72,-42,South East,England +E14001070,Ashton-under-Lyne,54,-23,North West,England +E14001071,Aylesbury,60,-35,South East,England +E14001072,Banbury,58,-33,South East,England +E14001073,Barking,68,-38,London,England +E14001074,Barnsley North,57,-23,Yorkshire and the Humber,England +E14001075,Barnsley South,58,-23,Yorkshire and the Humber,England +E14001076,Barrow and Furness,54,-16,North West,England +E14001077,Basildon and Billericay,67,-34,Eastern,England +E14001078,Basingstoke,55,-39,South East,England +E14001079,Bassetlaw,61,-26,East Midlands,England +E14001080,Bath,51,-40,South West,England +E14001081,Battersea,62,-41,London,England +E14001082,Beaconsfield,57,-37,South East,England +E14001083,Beckenham and Penge,65,-43,London,England +E14001084,Bedford,63,-32,Eastern,England +E14001085,Bermondsey and Old Southwark,64,-40,London,England +E14001086,Bethnal Green and Stepney,65,-39,London,England +E14001087,Beverley and Holderness,64,-22,Yorkshire and the Humber,England +E14001088,Bexhill and Battle,70,-44,South East,England +E14001089,Bexleyheath and Crayford,67,-39,London,England +E14001090,Bicester and Woodstock,59,-34,South East,England +E14001091,Birkenhead,49,-27,North West,England +E14001092,Birmingham Edgbaston,53,-33,West Midlands,England +E14001093,Birmingham Erdington,54,-31,West Midlands,England +E14001094,Birmingham Hall Green and Moseley,55,-32,West Midlands,England +E14001095,Birmingham Hodge Hill and Solihull North,55,-31,West Midlands,England +E14001096,Birmingham Ladywood,54,-32,West Midlands,England +E14001097,Birmingham Northfield,54,-34,West Midlands,England +E14001098,Birmingham Perry Barr,53,-31,West Midlands,England +E14001099,Birmingham Selly Oak,54,-33,West Midlands,England +E14001100,Birmingham Yardley,56,-32,West Midlands,England +E14001101,Bishop Auckland,54,-14,North East,England +E14001102,Blackburn,53,-19,North West,England +E14001103,Blackley and Middleton South,53,-23,North West,England +E14001104,Blackpool North and Fleetwood,53,-18,North West,England +E14001105,Blackpool South,52,-18,North West,England +E14001106,Blaydon and Consett,55,-14,North East,England +E14001107,Blyth and Ashington,55,-12,North East,England +E14001108,Bognor Regis and Littlehampton,63,-44,South East,England +E14001109,Bolsover,60,-26,East Midlands,England +E14001110,Bolton North East,52,-21,North West,England +E14001111,Bolton South and Walkden,52,-22,North West,England +E14001112,Bolton West,51,-21,North West,England +E14001113,Bootle,49,-22,North West,England +E14001114,Boston and Skegness,64,-26,East Midlands,England +E14001115,Bournemouth East,52,-43,South West,England +E14001116,Bournemouth West,52,-42,South West,England +E14001117,Bracknell,56,-39,South East,England +E14001118,Bradford East,58,-20,Yorkshire and the Humber,England +E14001119,Bradford South,56,-21,Yorkshire and the Humber,England +E14001120,Bradford West,57,-20,Yorkshire and the Humber,England +E14001121,Braintree,67,-31,Eastern,England +E14001122,Brent East,61,-38,London,England +E14001123,Brent West,60,-38,London,England +E14001124,Brentford and Isleworth,60,-40,London,England +E14001125,Brentwood and Ongar,66,-33,Eastern,England +E14001126,Bridgwater,48,-41,South West,England +E14001127,Bridlington and The Wolds,63,-20,Yorkshire and the Humber,England +E14001128,Brigg and Immingham,62,-24,Yorkshire and the Humber,England +E14001129,Brighton Kemptown and Peacehaven,67,-45,South East,England +E14001130,Brighton Pavilion,67,-44,South East,England +E14001131,Bristol Central,51,-38,South West,England +E14001132,Bristol East,52,-38,South West,England +E14001133,Bristol North East,51,-37,South West,England +E14001134,Bristol North West,50,-38,South West,England +E14001135,Bristol South,51,-39,South West,England +E14001136,Broadland and Fakenham,66,-27,Eastern,England +E14001137,Bromley and Biggin Hill,67,-42,London,England +E14001138,Bromsgrove,52,-33,West Midlands,England +E14001139,Broxbourne,66,-35,Eastern,England +E14001140,Broxtowe,59,-27,East Midlands,England +E14001141,Buckingham and Bletchley,60,-34,South East,England +E14001142,Burnley,55,-19,North West,England +E14001143,Burton and Uttoxeter,56,-28,West Midlands,England +E14001144,Bury North,53,-21,North West,England +E14001145,Bury South,53,-22,North West,England +E14001146,Bury St Edmunds and Stowmarket,68,-31,Eastern,England +E14001147,Calder Valley,56,-20,Yorkshire and the Humber,England +E14001148,Camborne and Redruth,43,-45,South West,England +E14001149,Cambridge,65,-30,Eastern,England +E14001150,Cannock Chase,54,-29,West Midlands,England +E14001151,Canterbury,71,-41,South East,England +E14001152,Carlisle,53,-14,North West,England +E14001153,Carshalton and Wallington,62,-43,London,England +E14001154,Castle Point,69,-36,Eastern,England +E14001155,Central Devon,47,-42,South West,England +E14001156,Central Suffolk and North Ipswich,68,-29,Eastern,England +E14001157,Chatham and Aylesford,69,-40,South East,England +E14001158,Cheadle,55,-26,North West,England +E14001159,Chelmsford,67,-33,Eastern,England +E14001160,Chelsea and Fulham,61,-40,London,England +E14001161,Cheltenham,52,-36,South West,England +E14001162,Chesham and Amersham,59,-36,South East,England +E14001163,Chester North and Neston,50,-28,North West,England +E14001164,Chester South and Eddisbury,51,-27,North West,England +E14001165,Chesterfield,59,-26,East Midlands,England +E14001166,Chichester,60,-44,South East,England +E14001167,Chingford and Woodford Green,64,-35,London,England +E14001168,Chippenham,52,-39,South West,England +E14001169,Chipping Barnet,62,-36,London,England +E14001170,Chorley,53,-20,North West,England +E14001171,Christchurch,53,-42,South West,England +E14001172,Cities of London and Westminster,63,-40,London,England +E14001173,City of Durham,55,-16,North East,England +E14001174,Clacton,69,-32,Eastern,England +E14001175,Clapham and Brixton Hill,62,-42,London,England +E14001176,Colchester,68,-32,Eastern,England +E14001177,Colne Valley,55,-23,Yorkshire and the Humber,England +E14001178,Congleton,54,-27,North West,England +E14001179,Corby and East Northamptonshire,62,-30,East Midlands,England +E14001180,Coventry East,57,-33,West Midlands,England +E14001181,Coventry North West,56,-33,West Midlands,England +E14001182,Coventry South,57,-34,West Midlands,England +E14001183,Cramlington and Killingworth,56,-12,North East,England +E14001184,Crawley,69,-44,South East,England +E14001185,Crewe and Nantwich,53,-27,North West,England +E14001186,Croydon East,65,-42,London,England +E14001187,Croydon South,64,-43,London,England +E14001188,Croydon West,63,-43,London,England +E14001189,Dagenham and Rainham,67,-37,London,England +E14001190,Darlington,55,-17,North East,England +E14001191,Dartford,68,-40,South East,England +E14001192,Daventry,60,-32,East Midlands,England +E14001193,Derby North,58,-28,East Midlands,England +E14001194,Derby South,57,-28,East Midlands,England +E14001195,Derbyshire Dales,57,-26,East Midlands,England +E14001196,Dewsbury and Batley,57,-22,Yorkshire and the Humber,England +E14001197,Didcot and Wantage,54,-38,South East,England +E14001198,Doncaster Central,60,-23,Yorkshire and the Humber,England +E14001199,Doncaster East and the Isle of Axholme,61,-23,Yorkshire and the Humber,England +E14001200,Doncaster North,61,-22,Yorkshire and the Humber,England +E14001201,Dorking and Horley,59,-43,South East,England +E14001202,Dover and Deal,72,-41,South East,England +E14001203,Droitwich and Evesham,54,-36,West Midlands,England +E14001204,Dudley,51,-31,West Midlands,England +E14001205,Dulwich and West Norwood,63,-42,London,England +E14001206,Dunstable and Leighton Buzzard,62,-33,Eastern,England +E14001207,Ealing Central and Acton,59,-39,London,England +E14001208,Ealing North,59,-38,London,England +E14001209,Ealing Southall,58,-39,London,England +E14001210,Earley and Woodley,56,-36,South East,England +E14001211,Easington,57,-16,North East,England +E14001212,East Grinstead and Uckfield,69,-43,South East,England +E14001213,East Ham,67,-38,London,England +E14001214,East Hampshire,55,-41,South East,England +E14001215,East Surrey,67,-43,South East,England +E14001216,East Thanet,71,-39,South East,England +E14001217,East Wiltshire,53,-41,South West,England +E14001218,East Worthing and Shoreham,65,-44,South East,England +E14001219,Eastbourne,69,-45,South East,England +E14001220,Eastleigh,54,-41,South East,England +E14001221,Edmonton and Winchmore Hill,64,-36,London,England +E14001222,Ellesmere Port and Bromborough,50,-27,North West,England +E14001223,Eltham and Chislehurst,66,-41,London,England +E14001224,Ely and East Cambridgeshire,66,-30,Eastern,England +E14001225,Enfield North,62,-35,London,England +E14001226,Epping Forest,67,-35,Eastern,England +E14001227,Epsom and Ewell,60,-43,South East,England +E14001228,Erewash,59,-28,East Midlands,England +E14001229,Erith and Thamesmead,67,-40,London,England +E14001230,Esher and Walton,58,-42,South East,England +E14001231,Exeter,48,-42,South West,England +E14001232,Exmouth and Exeter East,48,-43,South West,England +E14001233,Fareham and Waterlooville,55,-43,South East,England +E14001234,Farnham and Bordon,56,-42,South East,England +E14001235,Faversham and Mid Kent,71,-40,South East,England +E14001236,Feltham and Heston,59,-40,London,England +E14001237,Filton and Bradley Stoke,50,-37,South West,England +E14001238,Finchley and Golders Green,61,-37,London,England +E14001239,Folkestone and Hythe,71,-42,South East,England +E14001240,Forest of Dean,50,-35,South West,England +E14001241,Frome and East Somerset,50,-41,South West,England +E14001242,Fylde,51,-19,North West,England +E14001243,Gainsborough,61,-25,East Midlands,England +E14001244,Gateshead Central and Whickham,56,-15,North East,England +E14001245,Gedling,61,-28,East Midlands,England +E14001246,Gillingham and Rainham,70,-40,South East,England +E14001247,Glastonbury and Somerton,49,-41,South West,England +E14001248,Gloucester,51,-35,South West,England +E14001249,Godalming and Ash,57,-42,South East,England +E14001250,Goole and Pocklington,61,-21,Yorkshire and the Humber,England +E14001251,Gorton and Denton,55,-24,North West,England +E14001252,Gosport,57,-43,South East,England +E14001253,Grantham and Bourne,63,-28,East Midlands,England +E14001254,Gravesham,68,-39,South East,England +E14001255,Great Grimsby and Cleethorpes,63,-24,Yorkshire and the Humber,England +E14001256,Great Yarmouth,67,-27,Eastern,England +E14001257,Greenwich and Woolwich,66,-40,London,England +E14001258,Guildford,56,-41,South East,England +E14001259,Hackney North and Stoke Newington,64,-38,London,England +E14001260,Hackney South and Shoreditch,64,-39,London,England +E14001261,Halesowen,51,-33,West Midlands,England +E14001262,Halifax,55,-21,Yorkshire and the Humber,England +E14001263,Hamble Valley,56,-43,South East,England +E14001264,Hammersmith and Chiswick,60,-39,London,England +E14001265,Hampstead and Highgate,62,-38,London,England +E14001266,"Harborough, Oadby and Wigston",61,-31,East Midlands,England +E14001267,Harlow,67,-32,Eastern,England +E14001268,Harpenden and Berkhamsted,62,-34,Eastern,England +E14001269,Harrogate and Knaresborough,59,-18,Yorkshire and the Humber,England +E14001270,Harrow East,60,-37,London,England +E14001271,Harrow West,59,-37,London,England +E14001272,Hartlepool,59,-16,North East,England +E14001273,Harwich and North Essex,69,-31,Eastern,England +E14001274,Hastings and Rye,70,-43,South East,England +E14001275,Havant,59,-44,South East,England +E14001276,Hayes and Harlington,58,-38,London,England +E14001277,Hazel Grove,55,-25,North West,England +E14001278,Hemel Hempstead,64,-34,Eastern,England +E14001279,Hendon,61,-36,London,England +E14001280,Henley and Thame,58,-35,South East,England +E14001281,Hereford and South Herefordshire,51,-34,West Midlands,England +E14001282,Herne Bay and Sandwich,72,-40,South East,England +E14001283,Hertford and Stortford,66,-32,Eastern,England +E14001284,Hertsmere,66,-34,Eastern,England +E14001285,Hexham,53,-13,North East,England +E14001286,Heywood and Middleton North,54,-20,North West,England +E14001287,High Peak,56,-25,East Midlands,England +E14001288,Hinckley and Bosworth,58,-30,East Midlands,England +E14001289,Hitchin,64,-32,Eastern,England +E14001290,Holborn and St Pancras,62,-39,London,England +E14001291,Honiton and Sidmouth,49,-43,South West,England +E14001292,Hornchurch and Upminster,66,-37,London,England +E14001293,Hornsey and Friern Barnet,63,-36,London,England +E14001294,Horsham,62,-44,South East,England +E14001295,Houghton and Sunderland South,57,-15,North East,England +E14001296,Hove and Portslade,66,-44,South East,England +E14001297,Huddersfield,56,-22,Yorkshire and the Humber,England +E14001298,Huntingdon,63,-31,Eastern,England +E14001299,Hyndburn,54,-19,North West,England +E14001300,Ilford North,65,-36,London,England +E14001301,Ilford South,65,-37,London,England +E14001302,Ipswich,68,-30,Eastern,England +E14001303,Isle of Wight East,54,-45,South East,England +E14001304,Isle of Wight West,53,-45,South East,England +E14001305,Islington North,63,-38,London,England +E14001306,Islington South and Finsbury,63,-39,London,England +E14001307,Jarrow and Gateshead East,57,-14,North East,England +E14001308,Keighley and Ilkley,56,-19,Yorkshire and the Humber,England +E14001309,Kenilworth and Southam,56,-34,West Midlands,England +E14001310,Kensington and Bayswater,61,-39,London,England +E14001311,Kettering,61,-30,East Midlands,England +E14001312,Kingston and Surbiton,59,-42,London,England +E14001313,Kingston upon Hull East,63,-22,Yorkshire and the Humber,England +E14001314,Kingston upon Hull North and Cottingham,62,-21,Yorkshire and the Humber,England +E14001315,Kingston upon Hull West and Haltemprice,62,-22,Yorkshire and the Humber,England +E14001316,Kingswinford and South Staffordshire,52,-30,West Midlands,England +E14001317,Knowsley,50,-23,North West,England +E14001318,Lancaster and Wyre,54,-18,North West,England +E14001319,Leeds Central and Headingley,60,-20,Yorkshire and the Humber,England +E14001320,Leeds East,61,-20,Yorkshire and the Humber,England +E14001321,Leeds North East,59,-19,Yorkshire and the Humber,England +E14001322,Leeds North West,58,-19,Yorkshire and the Humber,England +E14001323,Leeds South,59,-21,Yorkshire and the Humber,England +E14001324,Leeds South West and Morley,58,-21,Yorkshire and the Humber,England +E14001325,Leeds West and Pudsey,59,-20,Yorkshire and the Humber,England +E14001326,Leicester East,60,-30,East Midlands,England +E14001327,Leicester South,60,-31,East Midlands,England +E14001328,Leicester West,59,-31,East Midlands,England +E14001329,Leigh and Atherton,51,-25,North West,England +E14001330,Lewes,68,-45,South East,England +E14001331,Lewisham East,66,-42,London,England +E14001332,Lewisham North,65,-40,London,England +E14001333,Lewisham West and East Dulwich,65,-41,London,England +E14001334,Leyton and Wanstead,64,-37,London,England +E14001335,Lichfield,56,-29,West Midlands,England +E14001336,Lincoln,62,-25,East Midlands,England +E14001337,Liverpool Garston,50,-25,North West,England +E14001338,Liverpool Riverside,49,-24,North West,England +E14001339,Liverpool Walton,49,-23,North West,England +E14001340,Liverpool Wavertree,49,-25,North West,England +E14001341,Liverpool West Derby,50,-24,North West,England +E14001342,Loughborough,59,-30,East Midlands,England +E14001343,Louth and Horncastle,63,-25,East Midlands,England +E14001344,Lowestoft,68,-28,Eastern,England +E14001345,Luton North,63,-33,Eastern,England +E14001346,Luton South and South Bedfordshire,63,-34,Eastern,England +E14001347,Macclesfield,56,-26,North West,England +E14001348,Maidenhead,57,-36,South East,England +E14001349,Maidstone and Malling,69,-41,South East,England +E14001350,Makerfield,51,-22,North West,England +E14001351,Maldon,69,-33,Eastern,England +E14001352,Manchester Central,54,-24,North West,England +E14001353,Manchester Rusholme,53,-25,North West,England +E14001354,Manchester Withington,54,-26,North West,England +E14001355,Mansfield,61,-27,East Midlands,England +E14001356,Melksham and Devizes,52,-40,South West,England +E14001357,Melton and Syston,61,-29,East Midlands,England +E14001358,Meriden and Solihull East,55,-33,West Midlands,England +E14001359,Mid Bedfordshire,62,-32,Eastern,England +E14001360,Mid Buckinghamshire,59,-35,South East,England +E14001361,Mid Cheshire,52,-27,North West,England +E14001362,Mid Derbyshire,57,-27,East Midlands,England +E14001363,Mid Dorset and North Poole,50,-43,South West,England +E14001364,Mid Leicestershire,58,-31,East Midlands,England +E14001365,Mid Norfolk,65,-28,Eastern,England +E14001366,Mid Sussex,68,-43,South East,England +E14001367,Middlesbrough and Thornaby East,57,-17,North East,England +E14001368,Middlesbrough South and East Cleveland,59,-17,North East,England +E14001369,Milton Keynes Central,61,-34,South East,England +E14001370,Milton Keynes North,61,-33,South East,England +E14001371,Mitcham and Morden,61,-43,London,England +E14001372,Morecambe and Lunesdale,54,-17,North West,England +E14001373,New Forest East,54,-43,South East,England +E14001374,New Forest West,53,-43,South East,England +E14001375,Newark,62,-26,East Midlands,England +E14001376,Newbury,54,-37,South East,England +E14001377,Newcastle upon Tyne Central and West,54,-13,North East,England +E14001378,Newcastle upon Tyne East and Wallsend,56,-14,North East,England +E14001379,Newcastle upon Tyne North,55,-13,North East,England +E14001380,Newcastle-under-Lyme,52,-28,West Midlands,England +E14001381,Newton Abbot,47,-43,South West,England +E14001382,Newton Aycliffe and Spennymoor,56,-16,North East,England +E14001383,Normanton and Hemsworth,59,-23,Yorkshire and the Humber,England +E14001384,North Bedfordshire,62,-31,Eastern,England +E14001385,North Cornwall,45,-43,South West,England +E14001386,North Cotswolds,53,-37,South West,England +E14001387,North Devon,46,-41,South West,England +E14001388,North Dorset,51,-42,South West,England +E14001389,North Durham,54,-15,North East,England +E14001390,North East Cambridgeshire,64,-29,Eastern,England +E14001391,North East Derbyshire,58,-26,East Midlands,England +E14001392,North East Hampshire,56,-38,South East,England +E14001393,North East Hertfordshire,65,-32,Eastern,England +E14001394,North East Somerset and Hanham,50,-39,South West,England +E14001395,North Herefordshire,52,-34,West Midlands,England +E14001396,North Norfolk,65,-27,Eastern,England +E14001397,North Northumberland,54,-12,North East,England +E14001398,North Shropshire,50,-29,West Midlands,England +E14001399,North Somerset,49,-39,South West,England +E14001400,North Warwickshire and Bedworth,57,-32,West Midlands,England +E14001401,North West Cambridgeshire,64,-30,Eastern,England +E14001402,North West Essex,66,-31,Eastern,England +E14001403,North West Hampshire,54,-39,South East,England +E14001404,North West Leicestershire,58,-29,East Midlands,England +E14001405,North West Norfolk,64,-28,Eastern,England +E14001406,Northampton North,61,-32,East Midlands,England +E14001407,Northampton South,60,-33,East Midlands,England +E14001408,Norwich North,66,-28,Eastern,England +E14001409,Norwich South,66,-29,Eastern,England +E14001410,Nottingham East,60,-29,East Midlands,England +E14001411,Nottingham North and Kimberley,60,-28,East Midlands,England +E14001412,Nottingham South,59,-29,East Midlands,England +E14001413,Nuneaton,57,-31,West Midlands,England +E14001414,Old Bexley and Sidcup,67,-41,London,England +E14001415,Oldham East and Saddleworth,55,-22,North West,England +E14001416,"Oldham West, Chadderton and Royton",54,-22,North West,England +E14001417,Orpington,66,-43,London,England +E14001418,Ossett and Denby Dale,58,-22,Yorkshire and the Humber,England +E14001419,Oxford East,58,-34,South East,England +E14001420,Oxford West and Abingdon,57,-35,South East,England +E14001421,Peckham,64,-41,London,England +E14001422,Pendle and Clitheroe,56,-18,North West,England +E14001423,Penistone and Stocksbridge,56,-23,Yorkshire and the Humber,England +E14001424,Penrith and Solway,52,-15,North West,England +E14001425,Peterborough,63,-29,Eastern,England +E14001426,Plymouth Moor View,46,-43,South West,England +E14001427,Plymouth Sutton and Devonport,47,-44,South West,England +E14001428,"Pontefract, Castleford and Knottingley",60,-22,Yorkshire and the Humber,England +E14001429,Poole,51,-43,South West,England +E14001430,Poplar and Limehouse,66,-39,London,England +E14001431,Portsmouth North,58,-43,South East,England +E14001432,Portsmouth South,58,-44,South East,England +E14001433,Preston,52,-19,North West,England +E14001434,Putney,61,-41,London,England +E14001435,Queen's Park and Maida Vale,62,-40,London,England +E14001436,Rawmarsh and Conisbrough,60,-24,Yorkshire and the Humber,England +E14001437,Rayleigh and Wickford,68,-34,Eastern,England +E14001438,Reading Central,55,-37,South East,England +E14001439,Reading West and Mid Berkshire,55,-36,South East,England +E14001440,Redcar,58,-17,North East,England +E14001441,Redditch,53,-35,West Midlands,England +E14001442,Reigate,68,-44,South East,England +E14001443,Ribble Valley,55,-18,North West,England +E14001444,Richmond and Northallerton,57,-18,Yorkshire and the Humber,England +E14001445,Richmond Park,59,-41,London,England +E14001446,Rochdale,54,-21,North West,England +E14001447,Rochester and Strood,69,-39,South East,England +E14001448,Romford,66,-36,London,England +E14001449,Romsey and Southampton North,54,-40,South East,England +E14001450,Rossendale and Darwen,55,-20,North West,England +E14001451,Rother Valley,60,-25,Yorkshire and the Humber,England +E14001452,Rotherham,59,-24,Yorkshire and the Humber,England +E14001453,Rugby,58,-32,West Midlands,England +E14001454,"Ruislip, Northwood and Pinner",60,-36,London,England +E14001455,Runcorn and Helsby,51,-28,North West,England +E14001456,Runnymede and Weybridge,57,-41,South East,England +E14001457,Rushcliffe,62,-28,East Midlands,England +E14001458,Rutland and Stamford,62,-29,East Midlands,England +E14001459,Salford,53,-24,North West,England +E14001460,Salisbury,52,-41,South West,England +E14001461,Scarborough and Whitby,61,-19,Yorkshire and the Humber,England +E14001462,Scunthorpe,61,-24,Yorkshire and the Humber,England +E14001463,Sefton Central,50,-20,North West,England +E14001464,Selby,60,-21,Yorkshire and the Humber,England +E14001465,Sevenoaks,68,-42,South East,England +E14001466,Sheffield Brightside and Hillsborough,58,-24,Yorkshire and the Humber,England +E14001467,Sheffield Central,58,-25,Yorkshire and the Humber,England +E14001468,Sheffield Hallam,57,-24,Yorkshire and the Humber,England +E14001469,Sheffield Heeley,57,-25,Yorkshire and the Humber,England +E14001470,Sheffield South East,59,-25,Yorkshire and the Humber,England +E14001471,Sherwood Forest,62,-27,East Midlands,England +E14001472,Shipley,57,-19,Yorkshire and the Humber,England +E14001473,Shrewsbury,51,-30,West Midlands,England +E14001474,Sittingbourne and Sheppey,70,-39,South East,England +E14001475,Skipton and Ripon,58,-18,Yorkshire and the Humber,England +E14001476,Sleaford and North Hykeham,63,-26,East Midlands,England +E14001477,Slough,56,-37,South East,England +E14001478,Smethwick,53,-32,West Midlands,England +E14001479,Solihull West and Shirley,55,-34,West Midlands,England +E14001480,South Basildon and East Thurrock,68,-36,Eastern,England +E14001481,South Cambridgeshire,65,-31,Eastern,England +E14001482,South Cotswolds,53,-38,South West,England +E14001483,South Derbyshire,57,-29,East Midlands,England +E14001484,South Devon,48,-45,South West,England +E14001485,South Dorset,51,-44,South West,England +E14001486,South East Cornwall,46,-44,South West,England +E14001487,South Holland and The Deepings,63,-27,East Midlands,England +E14001488,South Leicestershire,59,-32,East Midlands,England +E14001489,South Norfolk,67,-29,Eastern,England +E14001490,South Northamptonshire,59,-33,East Midlands,England +E14001491,South Ribble,52,-20,North West,England +E14001492,South Shields,58,-14,North East,England +E14001493,South Shropshire,50,-31,West Midlands,England +E14001494,South Suffolk,69,-30,Eastern,England +E14001495,South West Devon,47,-45,South West,England +E14001496,South West Hertfordshire,61,-35,Eastern,England +E14001497,South West Norfolk,65,-29,Eastern,England +E14001498,South West Wiltshire,51,-41,South West,England +E14001499,Southampton Itchen,55,-42,South East,England +E14001500,Southampton Test,54,-42,South East,England +E14001501,Southend East and Rochford,69,-34,Eastern,England +E14001502,Southend West and Leigh,68,-35,Eastern,England +E14001503,Southgate and Wood Green,63,-35,London,England +E14001504,Southport,50,-19,North West,England +E14001505,Spelthorne,58,-40,South East,England +E14001506,Spen Valley,57,-21,Yorkshire and the Humber,England +E14001507,St Albans,65,-34,Eastern,England +E14001508,St Austell and Newquay,45,-44,South West,England +E14001509,St Helens North,50,-21,North West,England +E14001510,St Helens South and Whiston,50,-22,North West,England +E14001511,St Ives,43,-46,South West,England +E14001512,St Neots and Mid Cambridgeshire,64,-31,Eastern,England +E14001513,Stafford,54,-28,West Midlands,England +E14001514,Staffordshire Moorlands,56,-27,West Midlands,England +E14001515,Stalybridge and Hyde,56,-24,North West,England +E14001516,Stevenage,64,-33,Eastern,England +E14001517,Stockport,54,-25,North West,England +E14001518,Stockton North,58,-16,North East,England +E14001519,Stockton West,56,-17,North East,England +E14001520,Stoke-on-Trent Central,55,-28,West Midlands,England +E14001521,Stoke-on-Trent North,55,-27,West Midlands,England +E14001522,Stoke-on-Trent South,55,-29,West Midlands,England +E14001523,"Stone, Great Wyrley and Penkridge",53,-28,West Midlands,England +E14001524,Stourbridge,51,-32,West Midlands,England +E14001525,Stratford and Bow,65,-38,London,England +E14001526,Stratford-on-Avon,54,-35,West Midlands,England +E14001527,Streatham and Croydon North,64,-42,London,England +E14001528,Stretford and Urmston,52,-24,North West,England +E14001529,Stroud,52,-37,South West,England +E14001530,Suffolk Coastal,69,-29,Eastern,England +E14001531,Sunderland Central,58,-15,North East,England +E14001532,Surrey Heath,57,-39,South East,England +E14001533,Sussex Weald,70,-42,South East,England +E14001534,Sutton and Cheam,60,-42,London,England +E14001535,Sutton Coldfield,56,-31,West Midlands,England +E14001536,Swindon North,53,-39,South West,England +E14001537,Swindon South,53,-40,South West,England +E14001538,Tamworth,57,-30,West Midlands,England +E14001539,Tatton,52,-26,North West,England +E14001540,Taunton and Wellington,49,-42,South West,England +E14001541,Telford,52,-29,West Midlands,England +E14001542,Tewkesbury,53,-36,South West,England +E14001543,The Wrekin,51,-29,West Midlands,England +E14001544,Thirsk and Malton,60,-18,Yorkshire and the Humber,England +E14001545,Thornbury and Yate,51,-36,South West,England +E14001546,Thurrock,67,-36,Eastern,England +E14001547,Tipton and Wednesbury,52,-31,West Midlands,England +E14001548,Tiverton and Minehead,47,-41,South West,England +E14001549,Tonbridge,68,-41,South East,England +E14001550,Tooting,61,-42,London,England +E14001551,Torbay,48,-44,South West,England +E14001552,Torridge and Tavistock,46,-42,South West,England +E14001553,Tottenham,62,-37,London,England +E14001554,Truro and Falmouth,44,-45,South West,England +E14001555,Tunbridge Wells,69,-42,South East,England +E14001556,Twickenham,58,-41,London,England +E14001557,Tynemouth,56,-13,North East,England +E14001558,Uxbridge and South Ruislip,58,-37,London,England +E14001559,Vauxhall and Camberwell Green,63,-41,London,England +E14001560,Wakefield and Rothwell,59,-22,Yorkshire and the Humber,England +E14001561,Wallasey,48,-27,North West,England +E14001562,Walsall and Bloxwich,55,-30,West Midlands,England +E14001563,Walthamstow,63,-37,London,England +E14001564,Warrington North,51,-23,North West,England +E14001565,Warrington South,51,-24,North West,England +E14001566,Warwick and Leamington,55,-35,West Midlands,England +E14001567,Washington and Gateshead South,55,-15,North East,England +E14001568,Watford,65,-35,Eastern,England +E14001569,Waveney Valley,67,-28,Eastern,England +E14001570,Weald of Kent,70,-41,South East,England +E14001571,Wellingborough and Rushden,63,-30,East Midlands,England +E14001572,Wells and Mendip Hills,50,-40,South West,England +E14001573,Welwyn Hatfield,65,-33,Eastern,England +E14001574,West Bromwich,52,-32,West Midlands,England +E14001575,West Dorset,50,-44,South West,England +E14001576,West Ham and Beckton,66,-38,London,England +E14001577,West Lancashire,49,-21,North West,England +E14001578,West Suffolk,67,-30,Eastern,England +E14001579,West Worcestershire,52,-35,West Midlands,England +E14001580,Westmorland and Lonsdale,53,-15,North West,England +E14001581,Weston-super-Mare,49,-40,South West,England +E14001582,Wetherby and Easingwold,62,-20,Yorkshire and the Humber,England +E14001583,Whitehaven and Workington,53,-16,North West,England +E14001584,Widnes and Halewood,51,-26,North West,England +E14001585,Wigan,51,-20,North West,England +E14001586,Wimbledon,60,-41,London,England +E14001587,Winchester,55,-40,South East,England +E14001588,Windsor,57,-38,South East,England +E14001589,Wirral West,49,-28,North West,England +E14001590,Witham,68,-33,Eastern,England +E14001591,Witney,56,-35,South East,England +E14001592,Woking,57,-40,South East,England +E14001593,Wokingham,55,-38,South East,England +E14001594,Wolverhampton North East,53,-29,West Midlands,England +E14001595,Wolverhampton South East,54,-30,West Midlands,England +E14001596,Wolverhampton West,53,-30,West Midlands,England +E14001597,Worcester,53,-34,West Midlands,England +E14001598,Worsley and Eccles,52,-23,North West,England +E14001599,Worthing West,64,-44,South East,England +E14001600,Wycombe,58,-36,South East,England +E14001601,Wyre Forest,50,-33,West Midlands,England +E14001602,Wythenshawe and Sale East,53,-26,North West,England +E14001603,Yeovil,50,-42,South West,England +E14001604,York Central,60,-19,Yorkshire and the Humber,England +E14001605,York Outer,61,-18,Yorkshire and the Humber,England +N05000001,Belfast East,45,-17,Northern Ireland,Northern Ireland +N05000002,Belfast North,45,-16,Northern Ireland,Northern Ireland +N05000003,Belfast South and Mid Down,45,-18,Northern Ireland,Northern Ireland +N05000004,Belfast West,44,-17,Northern Ireland,Northern Ireland +N05000005,East Antrim,45,-15,Northern Ireland,Northern Ireland +N05000006,East Londonderry,43,-15,Northern Ireland,Northern Ireland +N05000007,Fermanagh and South Tyrone,42,-17,Northern Ireland,Northern Ireland +N05000008,Foyle,42,-15,Northern Ireland,Northern Ireland +N05000009,Lagan Valley,44,-18,Northern Ireland,Northern Ireland +N05000010,Mid Ulster,43,-16,Northern Ireland,Northern Ireland +N05000011,Newry and Armagh,44,-19,Northern Ireland,Northern Ireland +N05000012,North Antrim,44,-15,Northern Ireland,Northern Ireland +N05000013,North Down,46,-16,Northern Ireland,Northern Ireland +N05000014,South Antrim,44,-16,Northern Ireland,Northern Ireland +N05000015,South Down,46,-18,Northern Ireland,Northern Ireland +N05000016,Strangford,46,-17,Northern Ireland,Northern Ireland +N05000017,Upper Bann,43,-18,Northern Ireland,Northern Ireland +N05000018,West Tyrone,42,-16,Northern Ireland,Northern Ireland +S14000021,East Renfrewshire,48,-11,Scotland,Scotland +S14000027,Na h-Eileanan an Iar,47,-2,Scotland,Scotland +S14000045,Midlothian,52,-11,Scotland,Scotland +S14000048,North Ayrshire and Arran,48,-10,Scotland,Scotland +S14000051,Orkney and Shetland,51,0,Scotland,Scotland +S14000060,Aberdeen North,52,-3,Scotland,Scotland +S14000061,Aberdeen South,52,-4,Scotland,Scotland +S14000062,Aberdeenshire North and Moray East,51,-3,Scotland,Scotland +S14000063,Airdrie and Shotts,50,-11,Scotland,Scotland +S14000064,Alloa and Grangemouth,50,-7,Scotland,Scotland +S14000065,Angus and Perthshire Glens,50,-5,Scotland,Scotland +S14000066,Arbroath and Broughty Ferry,52,-5,Scotland,Scotland +S14000067,"Argyll, Bute and South Lochaber",49,-5,Scotland,Scotland +S14000068,Bathgate and Linlithgow,51,-9,Scotland,Scotland +S14000069,"Caithness, Sutherland and Easter Ross",50,-2,Scotland,Scotland +S14000070,Coatbridge and Bellshill,50,-12,Scotland,Scotland +S14000071,Cowdenbeath and Kirkcaldy,52,-7,Scotland,Scotland +S14000072,Cumbernauld and Kirkintilloch,50,-8,Scotland,Scotland +S14000073,Dumfries and Galloway,51,-13,Scotland,Scotland +S14000074,"Dumfriesshire, Clydesdale and Tweeddale",52,-13,Scotland,Scotland +S14000075,Dundee Central,50,-6,Scotland,Scotland +S14000076,Dunfermline and Dollar,51,-7,Scotland,Scotland +S14000077,East Kilbride and Strathaven,48,-13,Scotland,Scotland +S14000078,Edinburgh East and Musselburgh,54,-10,Scotland,Scotland +S14000079,Edinburgh North and Leith,53,-9,Scotland,Scotland +S14000080,Edinburgh South,53,-10,Scotland,Scotland +S14000081,Edinburgh South West,52,-10,Scotland,Scotland +S14000082,Edinburgh West,52,-9,Scotland,Scotland +S14000083,Falkirk,51,-8,Scotland,Scotland +S14000084,Glasgow East,51,-10,Scotland,Scotland +S14000085,Glasgow North,49,-9,Scotland,Scotland +S14000086,Glasgow North East,50,-9,Scotland,Scotland +S14000087,Glasgow South,49,-11,Scotland,Scotland +S14000088,Glasgow South West,50,-10,Scotland,Scotland +S14000089,Glasgow West,49,-8,Scotland,Scotland +S14000090,Glenrothes and Mid Fife,52,-6,Scotland,Scotland +S14000091,Gordon and Buchan,50,-4,Scotland,Scotland +S14000092,Hamilton and Clyde Valley,51,-12,Scotland,Scotland +S14000093,Inverclyde and Renfrewshire West,48,-8,Scotland,Scotland +S14000094,"Inverness, Skye and West Ross-shire",49,-3,Scotland,Scotland +S14000095,Livingston,51,-11,Scotland,Scotland +S14000096,Lothian East,53,-11,Scotland,Scotland +S14000097,Mid Dunbartonshire,49,-7,Scotland,Scotland +S14000098,"Moray West, Nairn and Strathspey",49,-4,Scotland,Scotland +S14000099,"Motherwell, Wishaw and Carluke",52,-12,Scotland,Scotland +S14000100,North East Fife,51,-6,Scotland,Scotland +S14000101,Paisley and Renfrewshire North,48,-9,Scotland,Scotland +S14000102,Paisley and Renfrewshire South,49,-10,Scotland,Scotland +S14000103,Perth and Kinross-shire,51,-5,Scotland,Scotland +S14000104,Rutherglen,49,-12,Scotland,Scotland +S14000105,Stirling and Strathallan,49,-6,Scotland,Scotland +S14000106,West Dunbartonshire,48,-7,Scotland,Scotland +S14000107,"Ayr, Carrick and Cumnock",49,-13,Scotland,Scotland +S14000108,"Berwickshire, Roxburgh and Selkirk",53,-12,Scotland,Scotland +S14000109,Central Ayrshire,48,-12,Scotland,Scotland +S14000110,Kilmarnock and Loudoun,50,-13,Scotland,Scotland +S14000111,West Aberdeenshire and Kincardine,51,-4,Scotland,Scotland +W07000081,Aberafan Maesteg,46,-36,Wales,Wales +W07000082,Alyn and Deeside,49,-29,Wales,Wales +W07000083,Bangor Aberconwy,47,-31,Wales,Wales +W07000084,Blaenau Gwent and Rhymney,49,-33,Wales,Wales +W07000085,"Brecon, Radnor and Cwm Tawe",50,-32,Wales,Wales +W07000086,Bridgend,46,-37,Wales,Wales +W07000087,Caerfyrddin,49,-32,Wales,Wales +W07000088,Caerphilly,49,-35,Wales,Wales +W07000089,Cardiff East,48,-37,Wales,Wales +W07000090,Cardiff North,48,-36,Wales,Wales +W07000091,Cardiff South and Penarth,48,-38,Wales,Wales +W07000092,Cardiff West,47,-37,Wales,Wales +W07000093,Ceredigion Preseli,48,-34,Wales,Wales +W07000094,Clwyd East,49,-30,Wales,Wales +W07000095,Clwyd North,48,-30,Wales,Wales +W07000096,Dwyfor Meirionnydd,48,-31,Wales,Wales +W07000097,Gower,44,-37,Wales,Wales +W07000098,Llanelli,45,-36,Wales,Wales +W07000099,Merthyr Tydfil and Aberdare,49,-34,Wales,Wales +W07000100,Mid and South Pembrokeshire,44,-36,Wales,Wales +W07000101,Monmouthshire,50,-36,Wales,Wales +W07000102,Montgomeryshire and Glyndwr,49,-31,Wales,Wales +W07000103,Neath and Swansea East,47,-35,Wales,Wales +W07000104,Newport East,49,-37,Wales,Wales +W07000105,Newport West and Islwyn,49,-36,Wales,Wales +W07000106,Pontypridd,48,-35,Wales,Wales +W07000107,Rhondda and Ogmore,47,-36,Wales,Wales +W07000108,Swansea West,45,-37,Wales,Wales +W07000109,Torfaen,50,-34,Wales,Wales +W07000110,Vale of Glamorgan,47,-38,Wales,Wales +W07000111,Wrexham,50,-30,Wales,Wales +W07000112,Ynys Môn,46,-29,Wales,Wales diff --git a/src/policyengine/data/uk_geography/local_authorities_2023.csv b/src/policyengine/data/uk_geography/local_authorities_2023.csv new file mode 100644 index 000000000..ba0ccac24 --- /dev/null +++ b/src/policyengine/data/uk_geography/local_authorities_2023.csv @@ -0,0 +1,362 @@ +code,name,x,y +E06000001,Hartlepool,8,19 +E06000002,Middlesbrough,9,18 +E06000003,Redcar and Cleveland,9,19 +E06000004,Stockton-on-Tees,8,18 +E06000005,Darlington,7,18 +E06000006,Halton,1,11 +E06000007,Warrington,2,11 +E06000008,Blackburn with Darwen,4,15 +E06000009,Blackpool,2,15 +E06000010,"Kingston upon Hull, City of",10,15 +E06000011,East Riding of Yorkshire,11,16 +E06000012,North East Lincolnshire,11,14 +E06000013,North Lincolnshire,10,14 +E06000014,York,9,17 +E06000015,Derby,6,11 +E06000016,Leicester,8,8 +E06000017,Rutland,10,9 +E06000018,Nottingham,8,10 +E06000019,"Herefordshire, County of",0,8 +E06000020,Telford and Wrekin,2,9 +E06000021,Stoke-on-Trent,3,10 +E06000022,Bath and North East Somerset,1,3 +E06000023,"Bristol, City of",0,3 +E06000024,North Somerset,0,2 +E06000025,South Gloucestershire,1,4 +E06000026,Plymouth,-4,-2 +E06000027,Torbay,-3,-2 +E06000030,Swindon,2,4 +E06000031,Peterborough,11,9 +E06000032,Luton,10,7 +E06000033,Southend-on-Sea,16,6 +E06000034,Thurrock,15,4 +E06000035,Medway,15,1 +E06000036,Bracknell Forest,4,2 +E06000037,West Berkshire,2,2 +E06000038,Reading,2,3 +E06000039,Slough,6,4 +E06000040,Windsor and Maidenhead,4,3 +E06000041,Wokingham,3,3 +E06000042,Milton Keynes,6,5 +E06000043,Brighton and Hove,9,-2 +E06000044,Portsmouth,4,-1 +E06000045,Southampton,2,0 +E06000046,Isle of Wight,1,-2 +E06000047,County Durham,6,18 +E06000049,Cheshire East,4,11 +E06000050,Cheshire West and Chester,3,11 +E06000051,Shropshire,1,9 +E06000052,Cornwall,-5,-2 +E06000053,Isles of Scilly,-7,-3 +E06000054,Wiltshire,1,2 +E06000055,Bedford,9,7 +E06000056,Central Bedfordshire,9,6 +E06000057,Northumberland,5,20 +E06000058,"Bournemouth, Christchurch and Poole",0,0 +E06000059,Dorset,-1,0 +E06000060,Buckinghamshire,5,5 +E06000061,North Northamptonshire,9,9 +E06000062,West Northamptonshire,7,6 +E06000063,Cumberland,4,18 +E06000064,Westmorland and Furness,4,17 +E06000065,North Yorkshire,8,17 +E06000066,Somerset,0,1 +E07000008,Cambridge,12,8 +E07000009,East Cambridgeshire,12,9 +E07000010,Fenland,13,10 +E07000011,Huntingdonshire,10,8 +E07000012,South Cambridgeshire,11,8 +E07000032,Amber Valley,7,11 +E07000033,Bolsover,10,12 +E07000034,Chesterfield,9,12 +E07000035,Derbyshire Dales,7,12 +E07000036,Erewash,7,9 +E07000037,High Peak,7,13 +E07000038,North East Derbyshire,8,12 +E07000039,South Derbyshire,6,10 +E07000040,East Devon,-2,-1 +E07000041,Exeter,-3,-1 +E07000042,Mid Devon,-2,0 +E07000043,North Devon,-3,1 +E07000044,South Hams,-4,-3 +E07000045,Teignbridge,-2,-2 +E07000046,Torridge,-4,-1 +E07000047,West Devon,-3,0 +E07000061,Eastbourne,10,-2 +E07000062,Hastings,13,-2 +E07000063,Lewes,10,-1 +E07000064,Rother,12,-2 +E07000065,Wealden,11,-2 +E07000066,Basildon,14,5 +E07000067,Braintree,14,7 +E07000068,Brentwood,13,5 +E07000069,Castle Point,15,5 +E07000070,Chelmsford,14,6 +E07000071,Colchester,15,8 +E07000072,Epping Forest,12,5 +E07000073,Harlow,13,6 +E07000074,Maldon,15,7 +E07000075,Rochford,15,6 +E07000076,Tendring,16,8 +E07000077,Uttlesford,13,7 +E07000078,Cheltenham,1,5 +E07000079,Cotswold,2,5 +E07000080,Forest of Dean,-1,6 +E07000081,Gloucester,0,6 +E07000082,Stroud,0,5 +E07000083,Tewkesbury,1,6 +E07000084,Basingstoke and Deane,2,1 +E07000085,East Hampshire,4,0 +E07000086,Eastleigh,3,0 +E07000087,Fareham,2,-1 +E07000088,Gosport,3,-1 +E07000089,Hart,3,2 +E07000090,Havant,5,0 +E07000091,New Forest,1,0 +E07000092,Rushmoor,4,1 +E07000093,Test Valley,1,1 +E07000094,Winchester,3,1 +E07000095,Broxbourne,12,6 +E07000096,Dacorum,8,6 +E07000098,Hertsmere,9,5 +E07000099,North Hertfordshire,11,7 +E07000102,Three Rivers,7,5 +E07000103,Watford,8,5 +E07000105,Ashford,12,-1 +E07000106,Canterbury,15,0 +E07000107,Dartford,13,1 +E07000108,Dover,14,-1 +E07000109,Gravesham,14,1 +E07000110,Maidstone,14,0 +E07000111,Sevenoaks,12,0 +E07000112,Folkestone and Hythe,13,-1 +E07000113,Swale,16,0 +E07000114,Thanet,15,-1 +E07000115,Tonbridge and Malling,13,0 +E07000116,Tunbridge Wells,11,-1 +E07000117,Burnley,6,15 +E07000118,Chorley,3,14 +E07000119,Fylde,4,16 +E07000120,Hyndburn,5,15 +E07000121,Lancaster,3,17 +E07000122,Pendle,6,16 +E07000123,Preston,5,16 +E07000124,Ribble Valley,5,17 +E07000125,Rossendale,6,14 +E07000126,South Ribble,3,15 +E07000127,West Lancashire,2,13 +E07000128,Wyre,3,16 +E07000129,Blaby,7,7 +E07000130,Charnwood,8,9 +E07000131,Harborough,8,7 +E07000132,Hinckley and Bosworth,7,8 +E07000133,Melton,11,10 +E07000134,North West Leicestershire,6,9 +E07000135,Oadby and Wigston,9,8 +E07000136,Boston,12,12 +E07000137,East Lindsey,12,13 +E07000138,Lincoln,11,12 +E07000139,North Kesteven,11,11 +E07000140,South Holland,12,11 +E07000141,South Kesteven,12,10 +E07000142,West Lindsey,11,13 +E07000143,Breckland,14,10 +E07000144,Broadland,15,12 +E07000145,Great Yarmouth,15,11 +E07000146,King's Lynn and West Norfolk,13,11 +E07000147,North Norfolk,14,12 +E07000148,Norwich,14,11 +E07000149,South Norfolk,15,10 +E07000170,Ashfield,8,11 +E07000171,Bassetlaw,10,13 +E07000172,Broxtowe,7,10 +E07000173,Gedling,9,10 +E07000174,Mansfield,9,11 +E07000175,Newark and Sherwood,10,11 +E07000176,Rushcliffe,10,10 +E07000177,Cherwell,4,5 +E07000178,Oxford,4,4 +E07000179,South Oxfordshire,5,4 +E07000180,Vale of White Horse,3,4 +E07000181,West Oxfordshire,3,5 +E07000192,Cannock Chase,3,9 +E07000193,East Staffordshire,5,11 +E07000194,Lichfield,4,9 +E07000195,Newcastle-under-Lyme,2,10 +E07000196,South Staffordshire,2,8 +E07000197,Stafford,4,10 +E07000198,Staffordshire Moorlands,5,10 +E07000199,Tamworth,5,9 +E07000200,Babergh,14,8 +E07000202,Ipswich,15,9 +E07000203,Mid Suffolk,14,9 +E07000207,Elmbridge,7,2 +E07000208,Epsom and Ewell,8,0 +E07000209,Guildford,5,1 +E07000210,Mole Valley,6,1 +E07000211,Reigate and Banstead,7,0 +E07000212,Runnymede,5,3 +E07000213,Spelthorne,6,3 +E07000214,Surrey Heath,5,2 +E07000215,Tandridge,9,-1 +E07000216,Waverley,6,0 +E07000217,Woking,6,2 +E07000218,North Warwickshire,6,8 +E07000219,Nuneaton and Bedworth,6,7 +E07000220,Rugby,6,6 +E07000221,Stratford-on-Avon,3,6 +E07000222,Warwick,4,6 +E07000223,Adur,8,-2 +E07000224,Arun,6,-2 +E07000225,Chichester,5,-1 +E07000226,Crawley,8,-1 +E07000227,Horsham,6,-1 +E07000228,Mid Sussex,7,-1 +E07000229,Worthing,7,-2 +E07000234,Bromsgrove,2,7 +E07000235,Malvern Hills,-1,7 +E07000236,Redditch,4,7 +E07000237,Worcester,0,7 +E07000238,Wychavon,2,6 +E07000239,Wyre Forest,1,8 +E07000240,St Albans,10,6 +E07000241,Welwyn Hatfield,11,6 +E07000242,East Hertfordshire,13,8 +E07000243,Stevenage,12,7 +E07000244,East Suffolk,16,10 +E07000245,West Suffolk,13,9 +E08000001,Bolton,4,14 +E08000002,Bury,5,14 +E08000003,Manchester,5,12 +E08000004,Oldham,5,13 +E08000005,Rochdale,7,14 +E08000006,Salford,4,13 +E08000007,Stockport,6,12 +E08000008,Tameside,6,13 +E08000009,Trafford,4,12 +E08000010,Wigan,3,13 +E08000011,Knowsley,2,12 +E08000012,Liverpool,1,13 +E08000013,St. Helens,3,12 +E08000014,Sefton,2,14 +E08000015,Wirral,1,12 +E08000016,Barnsley,8,14 +E08000017,Doncaster,9,14 +E08000018,Rotherham,9,13 +E08000019,Sheffield,8,13 +E08000021,Newcastle upon Tyne,5,19 +E08000022,North Tyneside,6,20 +E08000023,South Tyneside,7,20 +E08000024,Sunderland,7,19 +E08000025,Birmingham,5,8 +E08000026,Coventry,5,6 +E08000027,Dudley,1,7 +E08000028,Sandwell,3,7 +E08000029,Solihull,5,7 +E08000030,Walsall,4,8 +E08000031,Wolverhampton,3,8 +E08000032,Bradford,7,16 +E08000033,Calderdale,7,15 +E08000034,Kirklees,8,15 +E08000035,Leeds,8,16 +E08000036,Wakefield,9,15 +E08000037,Gateshead,6,19 +E09000001,City of London,11,2 +E09000002,Barking and Dagenham,13,3 +E09000003,Barnet,10,5 +E09000004,Bexley,12,1 +E09000005,Brent,10,4 +E09000006,Bromley,11,0 +E09000007,Camden,11,4 +E09000008,Croydon,10,0 +E09000009,Ealing,9,4 +E09000010,Enfield,11,5 +E09000011,Greenwich,11,1 +E09000012,Hackney,12,3 +E09000013,Hammersmith and Fulham,8,3 +E09000014,Haringey,12,4 +E09000015,Harrow,8,4 +E09000016,Havering,14,3 +E09000017,Hillingdon,7,4 +E09000018,Hounslow,7,3 +E09000019,Islington,11,3 +E09000020,Kensington and Chelsea,9,3 +E09000021,Kingston upon Thames,7,1 +E09000022,Lambeth,10,2 +E09000023,Lewisham,10,1 +E09000024,Merton,8,1 +E09000025,Newham,13,2 +E09000026,Redbridge,14,4 +E09000027,Richmond upon Thames,8,2 +E09000028,Southwark,9,1 +E09000029,Sutton,9,0 +E09000030,Tower Hamlets,12,2 +E09000031,Waltham Forest,13,4 +E09000032,Wandsworth,9,2 +E09000033,Westminster,10,3 +N09000001,Antrim and Newtownabbey,-4,16 +N09000002,"Armagh City, Banbridge and Craigavon",-5,16 +N09000003,Belfast,-4,17 +N09000004,Causeway Coast and Glens,-5,18 +N09000005,Derry City and Strabane,-6,17 +N09000006,Fermanagh and Omagh,-6,16 +N09000007,Lisburn and Castlereagh,-5,15 +N09000008,Mid and East Antrim,-4,18 +N09000009,Mid Ulster,-5,17 +N09000010,"Newry, Mourne and Down",-4,15 +N09000011,Ards and North Down,-3,16 +S12000005,Clackmannanshire,2,24 +S12000006,Dumfries and Galloway,4,20 +S12000008,East Ayrshire,3,20 +S12000010,East Lothian,5,22 +S12000011,East Renfrewshire,2,20 +S12000013,Na h-Eileanan Siar,-1,27 +S12000014,Falkirk,2,23 +S12000017,Highland,1,26 +S12000018,Inverclyde,0,21 +S12000019,Midlothian,3,21 +S12000020,Moray,2,26 +S12000021,North Ayrshire,1,20 +S12000023,Orkney Islands,4,28 +S12000026,Scottish Borders,4,21 +S12000027,Shetland Islands,5,30 +S12000028,South Ayrshire,1,19 +S12000029,South Lanarkshire,2,21 +S12000030,Stirling,1,24 +S12000033,Aberdeen City,4,26 +S12000034,Aberdeenshire,3,26 +S12000035,Argyll and Bute,0,24 +S12000036,City of Edinburgh,4,22 +S12000038,Renfrewshire,1,22 +S12000039,West Dunbartonshire,0,23 +S12000040,West Lothian,3,22 +S12000041,Angus,2,25 +S12000042,Dundee City,3,25 +S12000045,East Dunbartonshire,1,23 +S12000047,Fife,3,24 +S12000048,Perth and Kinross,1,25 +S12000049,Glasgow City,1,21 +S12000050,North Lanarkshire,2,22 +W06000001,Isle of Anglesey,-2,12 +W06000002,Gwynedd,-2,10 +W06000003,Conwy,-1,10 +W06000004,Denbighshire,0,10 +W06000005,Flintshire,0,11 +W06000006,Wrexham,1,10 +W06000008,Ceredigion,-2,9 +W06000009,Pembrokeshire,-5,6 +W06000010,Carmarthenshire,-4,6 +W06000011,Swansea,-4,5 +W06000012,Neath Port Talbot,-3,5 +W06000013,Bridgend,-3,6 +W06000014,Vale of Glamorgan,-2,4 +W06000015,Cardiff,-2,5 +W06000016,Rhondda Cynon Taf,-3,7 +W06000018,Caerphilly,-2,6 +W06000019,Blaenau Gwent,0,9 +W06000020,Torfaen,-2,7 +W06000021,Monmouthshire,-1,8 +W06000022,Newport,-1,5 +W06000023,Powys,-1,9 +W06000024,Merthyr Tydfil,-2,8 diff --git a/src/policyengine/data/uk_geography_assets.py b/src/policyengine/data/uk_geography_assets.py index 14257314c..d4d6bf788 100644 --- a/src/policyengine/data/uk_geography_assets.py +++ b/src/policyengine/data/uk_geography_assets.py @@ -2,6 +2,7 @@ import os from dataclasses import dataclass +from importlib.resources import files from pathlib import Path from typing import Optional, Sequence, Union @@ -51,6 +52,29 @@ class UKGeographyAssetPaths: lookup_csv_filename="local_authorities_2021.csv", ) +# Area lists shipped with the package: code, name and hex-map position for +# every constituency (2024 boundaries) and local authority (April 2023 +# districts). They define the constituency and local-authority regions and +# label geography outputs. They are not the lookup CSVs paired with the weight +# matrices above: a matrix's paired CSV gives each area its row, so it must +# stay the file the matrix was built with. See data/uk_geography/SOURCES.md. +UK_AREA_LIST_FILES = { + "constituency": "constituencies_2024.csv", + "local_authority": "local_authorities_2023.csv", +} + + +def packaged_uk_area_list_path(geography_type: str) -> str: + """Return the packaged area list for a UK geography type.""" + try: + filename = UK_AREA_LIST_FILES[geography_type] + except KeyError: + raise ValueError( + f"No packaged UK area list for geography type {geography_type!r}. " + f"Known types: {sorted(UK_AREA_LIST_FILES)}" + ) from None + return str(files("policyengine").joinpath("data", "uk_geography", filename)) + def _env_path(name: str) -> Optional[Path]: value = os.environ.get(name) diff --git a/src/policyengine/outputs/constituency_impact.py b/src/policyengine/outputs/constituency_impact.py index 214664a65..dfd32d375 100644 --- a/src/policyengine/outputs/constituency_impact.py +++ b/src/policyengine/outputs/constituency_impact.py @@ -79,13 +79,14 @@ def compute_uk_constituency_impacts( reform_simulation: Completed reform simulation. weight_matrix_path: Deprecated and ignored. Constituency outputs now group by ``constituency_code_oa`` on the household output. - constituency_csv_path: Optional path to constituencies_2024.csv. - If omitted, standard local paths are checked before downloading - from GCS. If still unavailable, results use geography codes as names. + constituency_csv_path: Optional path to a lookup CSV with ``code``, + ``name``, ``x`` and ``y`` columns. If omitted, the packaged 2024 + constituency list supplies names and hex positions. Codes missing + from the lookup use the code as the name. year: Deprecated and ignored. asset_strategies: Deprecated and ignored. - download_missing_assets: Whether to download the optional lookup CSV - from GCS when no local CSV is found. + download_missing_assets: Deprecated and ignored; the lookup is + packaged. Returns: ConstituencyImpact with constituency_results populated. diff --git a/src/policyengine/outputs/local_authority_impact.py b/src/policyengine/outputs/local_authority_impact.py index e38b48d13..fc6957d6b 100644 --- a/src/policyengine/outputs/local_authority_impact.py +++ b/src/policyengine/outputs/local_authority_impact.py @@ -79,13 +79,15 @@ def compute_uk_local_authority_impacts( reform_simulation: Completed reform simulation. weight_matrix_path: Deprecated and ignored. Local-authority outputs now group by ``la_code_oa`` on the household output. - local_authority_csv_path: Optional path to local_authorities_2021.csv. - If omitted, standard local paths are checked before downloading - from GCS. If still unavailable, results use geography codes as names. + local_authority_csv_path: Optional path to a lookup CSV with + ``code``, ``name``, ``x`` and ``y`` columns. If omitted, the + packaged April 2023 local-authority list supplies names and + hex positions. Codes missing from the lookup use the code as + the name. year: Deprecated and ignored. asset_strategies: Deprecated and ignored. - download_missing_assets: Whether to download the optional lookup CSV - from GCS when no local CSV is found. + download_missing_assets: Deprecated and ignored; the lookup is + packaged. Returns: LocalAuthorityImpact with local_authority_results populated. diff --git a/src/policyengine/outputs/uk_geography_impact.py b/src/policyengine/outputs/uk_geography_impact.py index 6a283c63a..f6158d722 100644 --- a/src/policyengine/outputs/uk_geography_impact.py +++ b/src/policyengine/outputs/uk_geography_impact.py @@ -8,9 +8,11 @@ import pandas as pd from policyengine.data.uk_geography_assets import ( + UK_AREA_LIST_FILES, UKGeographyAssetSpec, default_download_dir, default_local_search_dirs, + packaged_uk_area_list_path, ) logger = logging.getLogger(__name__) @@ -45,7 +47,14 @@ def resolve_uk_geography_lookup_csv_path( lookup_csv_path: Optional[str] = None, download_missing_assets: bool = True, ) -> Optional[str]: - """Resolve a UK geography lookup CSV without requiring a weight matrix.""" + """Resolve a UK geography lookup CSV without requiring a weight matrix. + + An explicit ``lookup_csv_path`` wins. Constituencies and local + authorities otherwise use the area list shipped with the package, so + labels never depend on a local file or on the unversioned GCS bucket + (``download_missing_assets`` then has no effect). Other geography types + keep the local search and GCS fallback. + """ if lookup_csv_path: path = Path(lookup_csv_path).expanduser() if not path.is_file(): @@ -55,6 +64,9 @@ def resolve_uk_geography_lookup_csv_path( ) return str(path) + if spec.geography_type in UK_AREA_LIST_FILES: + return packaged_uk_area_list_path(spec.geography_type) + for search_dir in default_local_search_dirs(): path = search_dir / spec.lookup_csv_filename if path.is_file(): diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index e31762f96..9594b4f02 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -20,13 +20,15 @@ data_producer="populace", manifest_uri=( "hf://dataset/policyengine/populace-uk-private" - "@populace-uk-2023-0cdbb27-c239dfe51c11-20260615T201302Z" - "/releases/populace-uk-2023-0cdbb27-c239dfe51c11-20260615T201302Z" - "/release_manifest.json" + "@microcosm-uk-2024-25-national-20261002T230158Z-5c6b3f68" + "/releases/microcosm-uk-2024-25-national/release_manifest.json" ), ) print(result.summary()) +Pin the cut tag (``@``), never a branch: a release line's directory +(``releases//``) is rewritten by each cut. + ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose data release predates release manifests need a dedicated data-producer strategy @@ -46,11 +48,17 @@ import requests +from policyengine.data.uk_geography_assets import ( + CONSTITUENCY_ASSET_SPEC, + LOCAL_AUTHORITY_ASSET_SPEC, +) from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, DataReleaseManifest, + RegionalReleaseKind, _specifier_matches, fetch_pypi_wheel_metadata, + release_build_id, ) MANIFEST_URI_PATTERN = re.compile( @@ -82,6 +90,29 @@ # source-enrichment releases; `populace-data` is its earlier name. Both denote # the same producer, so every Populace-specific rule below must accept either. POPULACE_DATA_PACKAGES = frozenset({"populace-data", "microcosm-data"}) +# The Hugging Face repos each country's Populace/Microcosm releases publish to. +POPULACE_RELEASE_REPOS = { + "us": "policyengine/populace-us", + "uk": "policyengine/populace-uk-private", +} +# Region types a regional release serves. Their regions load the routed dataset +# and keep row-filtering it on their own geography column. +LOCAL_AREA_REGION_TYPES = { + "us": ("state", "congressional_district"), + "uk": ("constituency", "local_authority"), +} +REGIONAL_RELEASE_KINDS: tuple[RegionalReleaseKind, ...] = ( + "local_area", + "area_coded_national", +) +# A UK area-coded national release (the enhanced FRS) serves constituency and +# local-authority runs with these weight matrices as well as its households. +# Consumers bind a matrix only to the release of the dataset it reweights, so +# they travel with that dataset. +UK_AREA_WEIGHT_MATRIX_PATHS = ( + CONSTITUENCY_ASSET_SPEC.weight_matrix_filename, + LOCAL_AUTHORITY_ASSET_SPEC.weight_matrix_filename, +) # Artifact kinds published beside the release manifest rather than at the repo # root, so their manifest paths are relative to the release directory. RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) @@ -271,19 +302,8 @@ def validate_release_manifest( if not artifact.sha256: warnings.append(f"artifact {name!r} has no sha256") - built_with = None - if manifest.build is not None and manifest.build.built_with_model_package: - build_pkg = manifest.build.built_with_model_package - if build_pkg.name == model_package: - built_with = build_pkg.version - - claim_specifiers = [ - package.specifier - for package in manifest.compatible_model_packages - if package.name == model_package - ] - claim_matches = any( - _specifier_matches(model_version, specifier) for specifier in claim_specifiers + built_with, claim_specifiers, claim_matches = _model_claims( + manifest, model_package, model_version ) core_rests_on_qualification = core_version is not None and ( @@ -332,6 +352,64 @@ def validate_release_manifest( ) +def _model_claims( + manifest: DataReleaseManifest, + model_package: str, + model_version: str, +) -> tuple[Optional[str], list[str], bool]: + """A release's build-time model version and its claims on ``model_version``. + + Returns ``(built_with, claim_specifiers, claim_matches)``. + """ + built_with = None + if manifest.build is not None and manifest.build.built_with_model_package: + build_pkg = manifest.build.built_with_model_package + if build_pkg.name == model_package: + built_with = build_pkg.version + + claim_specifiers = [ + package.specifier + for package in manifest.compatible_model_packages + if package.name == model_package + ] + claim_matches = any( + _specifier_matches(model_version, specifier) for specifier in claim_specifiers + ) + return built_with, claim_specifiers, claim_matches + + +def model_compatibility_basis( + manifest: DataReleaseManifest, + model_package: str, + model_version: str, +) -> tuple[str, list[str]]: + """Return the basis on which a release serves a model version by itself. + + The release must be built with ``model_version`` or claim compatibility + with it; automated qualification of a pair does not apply. Certification + uses this for a regional release, which no qualification names. Returns + ``(compatibility_basis, warnings)``. + """ + built_with, claim_specifiers, claim_matches = _model_claims( + manifest, model_package, model_version + ) + if built_with == model_version: + return BASIS_BUILT_WITH, [] + if claim_matches: + return BASIS_PUBLISHER_CLAIM, [ + f"certifying {model_package} {model_version} on the publisher's " + f"compatibility claim {claim_specifiers}; the data was built " + f"with {built_with or 'an unrecorded model version'} — the test " + "suite is the arbiter" + ] + raise CertificationError( + f"{model_package} {model_version} matches neither the build-time " + f"model ({built_with or 'unrecorded'}) nor any publisher " + f"compatibility claim {claim_specifiers or '(none declared)'}; " + "a new data build or a published compatibility claim is required." + ) + + def _check_core_compatibility( manifest: DataReleaseManifest, core_version: str, @@ -477,11 +555,9 @@ def should_validate_vendored_artifacts( manifest: DataReleaseManifest, uri_parts: dict, ) -> bool: - return ( - country == "us" - and manifest.data_package.name in POPULACE_DATA_PACKAGES - and uri_parts["repo_id"] == "policyengine/populace-us" - ) + return manifest.data_package.name in POPULACE_DATA_PACKAGES and uri_parts[ + "repo_id" + ] == POPULACE_RELEASE_REPOS.get(country) def _state_code_from_path(path: str, artifact_prefix: str) -> Optional[str]: @@ -560,29 +636,87 @@ def merge_us_state_release_manifest( return DataReleaseManifest.model_validate(primary_payload) -def merge_us_local_area_release_manifest( +def _local_area_region_types(country: str) -> tuple[str, ...]: + try: + return LOCAL_AREA_REGION_TYPES[country] + except KeyError: + raise CertificationError( + f"Regional data releases are not supported for country {country!r}." + ) from None + + +def _route_region_types( + country: str, + primary_manifest: DataReleaseManifest, + primary_payload: dict, + path: str, + release_label: str, +) -> None: + """Point the country's local region types at ``path`` in the payload.""" + metadata = primary_payload.setdefault("metadata", {}) + region_datasets = metadata.setdefault("region_datasets", {}) + default_dataset = primary_manifest.default_datasets["national"] + default_artifact = primary_manifest.artifacts[default_dataset] + region_datasets.setdefault( + "national", + {"path_template": default_artifact.path}, + ) + regional_template = {"path_template": path} + for region_type in _local_area_region_types(country): + existing = region_datasets.get(region_type) + if existing is not None and existing != regional_template: + raise CertificationError( + f"{release_label} conflicts with the {region_type!r} " + "dataset template in the primary manifest." + ) + region_datasets[region_type] = regional_template + + +def _add_regional_artifact( + merged_artifacts: dict, + name: str, + artifact, + regional_repo_type: Optional[Literal["model", "dataset"]], +) -> None: + if name in merged_artifacts: + raise CertificationError( + f"Regional artifact {name!r} conflicts with the primary manifest." + ) + if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + raise CertificationError( + f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + ) + artifact_payload = artifact.model_dump(mode="json", exclude_none=True) + if regional_repo_type is not None: + artifact_payload["repo_type"] = regional_repo_type + merged_artifacts[name] = artifact_payload + + +def merge_local_area_release_manifest( + country: str, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, regional_repo_type: Optional[Literal["model", "dataset"]] = None, ) -> DataReleaseManifest: - """Add one certified local-area dataset to a primary US release. + """Add one certified local-area dataset to a primary release. - The supplemental release remains non-default nationally. State and - congressional-district simulations load the shared artifact and apply the - existing row-filtering strategy at runtime. + The supplemental release remains non-default nationally. The country's + local region types (US states and congressional districts, UK + constituencies and local authorities) load the shared artifact and apply + their existing row-filtering strategy at runtime. """ + label = f"{country.upper()} local-area release" + _local_area_region_types(country) if regional_manifest.dataset_role != "non_default_local_area": raise CertificationError( - "US local-area release must declare dataset_role='non_default_local_area'." + f"{label} must declare dataset_role='non_default_local_area'." ) if regional_manifest.is_default is not False: - raise CertificationError("US local-area release must declare is_default=false.") + raise CertificationError(f"{label} must declare is_default=false.") if regional_manifest.default_datasets: - raise CertificationError( - "US local-area release must not declare default datasets." - ) + raise CertificationError(f"{label} must not declare default datasets.") local_area_artifacts = [ (name, artifact) @@ -591,82 +725,200 @@ def merge_us_local_area_release_manifest( ] if len(local_area_artifacts) != 1: raise CertificationError( - "US local-area release must contain exactly one microdata artifact." + f"{label} must contain exactly one microdata artifact." ) artifact_name, artifact = local_area_artifacts[0] if not artifact.path.endswith(".h5"): - raise CertificationError("US local-area artifact must be an H5 file.") + raise CertificationError( + f"{country.upper()} local-area artifact must be an H5 file." + ) if not artifact.repo_id or not artifact.revision or not artifact.sha256: raise CertificationError( - "US local-area artifact must declare repo_id, revision, and sha256." + f"{country.upper()} local-area artifact must declare repo_id, " + "revision, and sha256." ) primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) merged_artifacts = primary_payload.setdefault("artifacts", {}) - if artifact_name in merged_artifacts: + _add_regional_artifact( + merged_artifacts, artifact_name, artifact, regional_repo_type + ) + _route_region_types( + country, primary_manifest, primary_payload, artifact.path, label + ) + + return DataReleaseManifest.model_validate(primary_payload) + + +def merge_us_local_area_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, +) -> DataReleaseManifest: + """Add one certified local-area dataset to a primary US release.""" + + return merge_local_area_release_manifest( + "us", + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ) + + +def merge_area_coded_national_release_manifest( + country: str, + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, +) -> DataReleaseManifest: + """Route UK area regions to another release's national dataset. + + This serves a national-first switch: the primary release's national + dataset carries no area codes, and the regional manifest is a certified + national release (the enhanced FRS) whose households do. Its national + dataset and its area weight matrices join the certified datasets with + their own repo and revision pins, and constituency and local-authority + regions load that dataset. The regional release's own default stays + non-default here. + """ + + label = f"{country.upper()} area-coded national release" + if country != "uk": raise CertificationError( - f"Regional artifact {artifact_name!r} conflicts with the primary manifest." + "Area-coded national regional releases are only supported for the UK." ) - if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + if regional_manifest.dataset_role not in (None, "national_default"): raise CertificationError( - f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + f"{label} must be a national release, not " + f"dataset_role={regional_manifest.dataset_role!r}. Certify a " + "local-area release with --regional-release-kind local_area." + ) + area_dataset = regional_manifest.default_datasets.get("national") + if area_dataset is None or area_dataset not in regional_manifest.artifacts: + raise CertificationError( + f"{label} must declare a national default dataset among its artifacts." + ) + area_artifact = regional_manifest.artifacts[area_dataset] + if area_artifact.kind != "microdata" or not area_artifact.path.endswith(".h5"): + raise CertificationError( + f"{label} national dataset {area_dataset!r} must be an H5 microdata " + "artifact." ) - artifact_payload = artifact.model_dump( - mode="json", - exclude_none=True, - ) - if regional_repo_type is not None: - artifact_payload["repo_type"] = regional_repo_type - merged_artifacts[artifact_name] = artifact_payload - metadata = primary_payload.setdefault("metadata", {}) - region_datasets = metadata.setdefault("region_datasets", {}) - default_dataset = primary_manifest.default_datasets["national"] - default_artifact = primary_manifest.artifacts[default_dataset] - region_datasets.setdefault( - "national", - {"path_template": default_artifact.path}, - ) - regional_template = {"path_template": artifact.path} - for region_type in ("state", "congressional_district"): - existing = region_datasets.get(region_type) - if existing is not None and existing != regional_template: + carried = {area_dataset: area_artifact} + weights = { + name: artifact + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "weights" + } + weight_paths = {artifact.path for artifact in weights.values()} + missing = [path for path in UK_AREA_WEIGHT_MATRIX_PATHS if path not in weight_paths] + if missing: + raise CertificationError( + f"{label} must carry the area weight matrices {missing}." + ) + carried.update(weights) + for name, artifact in carried.items(): + if not artifact.repo_id or not artifact.revision or not artifact.sha256: raise CertificationError( - f"US local-area release conflicts with the {region_type!r} " - "dataset template in the primary manifest." + f"{label} artifact {name!r} must declare repo_id, revision, and sha256." ) - region_datasets[region_type] = regional_template + + primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) + merged_artifacts = primary_payload.setdefault("artifacts", {}) + for name, artifact in carried.items(): + _add_regional_artifact(merged_artifacts, name, artifact, regional_repo_type) + _route_region_types( + country, primary_manifest, primary_payload, area_artifact.path, label + ) return DataReleaseManifest.model_validate(primary_payload) -def merge_us_regional_release_manifest( +def merge_regional_release_manifest( + country: str, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, + release_kind: RegionalReleaseKind = "local_area", regional_repo_type: Optional[Literal["model", "dataset"]] = None, artifact_prefix: str = "states/", path_template: str = "states/{state_code}.h5", -) -> DataReleaseManifest: - """Merge a typed shared local-area release or a legacy per-state release.""" +) -> tuple[DataReleaseManifest, Optional[RegionalReleaseKind]]: + """Merge a regional release into a primary release. + Returns the merged manifest and the routing kind to record. The kind is + ``None`` only for a legacy US per-state release. + """ + + if release_kind not in REGIONAL_RELEASE_KINDS: + raise CertificationError( + f"Unknown regional release kind {release_kind!r}; expected one of " + f"{list(REGIONAL_RELEASE_KINDS)}." + ) + if release_kind == "area_coded_national": + return ( + merge_area_coded_national_release_manifest( + country, + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ), + release_kind, + ) if regional_manifest.dataset_role == "non_default_local_area": - return merge_us_local_area_release_manifest( - primary_manifest, - regional_manifest, - regional_repo_type=regional_repo_type, + return ( + merge_local_area_release_manifest( + country, + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ), + "local_area", ) - if regional_manifest.dataset_role is not None: + if country == "us" and regional_manifest.dataset_role is None: + return ( + merge_us_state_release_manifest( + primary_manifest, + regional_manifest, + artifact_prefix=artifact_prefix, + path_template=path_template, + ), + None, + ) + if country == "uk" and regional_manifest.dataset_role in (None, "national_default"): raise CertificationError( - "Unsupported US regional release dataset_role " - f"{regional_manifest.dataset_role!r}." + "UK regional release is a national release, not a local-area one. " + "To route constituency and local-authority regions to its national " + "dataset, pass --regional-release-kind area_coded_national." ) - return merge_us_state_release_manifest( + raise CertificationError( + f"Unsupported {country.upper()} regional release dataset_role " + f"{regional_manifest.dataset_role!r}." + ) + + +def merge_us_regional_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, + artifact_prefix: str = "states/", + path_template: str = "states/{state_code}.h5", +) -> DataReleaseManifest: + """Merge a typed shared local-area release or a legacy per-state release.""" + + manifest, _ = merge_regional_release_manifest( + "us", primary_manifest, regional_manifest, + regional_repo_type=regional_repo_type, artifact_prefix=artifact_prefix, path_template=path_template, ) + return manifest def build_country_manifest_payload( @@ -760,9 +1012,10 @@ def build_country_manifest_payload( certified_artifact["sha256"] = default_artifact.sha256 build = manifest.build if build is not None: - if build.build_id: - certification["data_build_id"] = build.build_id - certified_artifact["build_id"] = build.build_id + build_id = release_build_id(build) + if build_id: + certification["data_build_id"] = build_id + certified_artifact["build_id"] = build_id model_build = build.built_with_model_package if model_build is not None: certification["built_with_model_version"] = model_build.version @@ -808,6 +1061,7 @@ def build_bundle_data_release_payload( uri_parts: dict, regional_manifest_uri: Optional[str] = None, regional_uri_parts: Optional[dict] = None, + regional_release_kind: Optional[RegionalReleaseKind] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -825,6 +1079,8 @@ def build_bundle_data_release_payload( payload["regional_release_manifest_uri"] = https_manifest_url( regional_uri_parts ) + if regional_release_kind is not None: + payload["regional_release_kind"] = regional_release_kind payload["default_dataset_uri"] = certified_artifact.get("uri") return payload @@ -844,6 +1100,7 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, ) -> tuple[dict, str, list[str]]: @@ -865,6 +1122,7 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, ) -> tuple[dict, str, list[str]]: @@ -889,6 +1147,7 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, ) -> tuple[dict, str, list[str]]: @@ -896,22 +1155,41 @@ def certify( manifest_uri, token=token ) regional_uri_parts = None + recorded_release_kind = None + regional_artifacts: dict = {} + regional_warnings: list[str] = [] if regional_manifest_uri is not None: - if country != "us": - raise CertificationError( - "Regional data release manifests are only supported for US " - "Populace certification." - ) regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token ) - manifest = merge_us_regional_release_manifest( + # The regional release serves the same model, so it must make its + # own compatibility claim for the pinned version. + try: + _, regional_warnings = model_compatibility_basis( + regional_manifest, model_package, model_version + ) + except CertificationError as exc: + raise CertificationError( + f"Regional release {regional_manifest_uri}: {exc}" + ) from exc + regional_warnings = [ + f"regional release: {warning}" for warning in regional_warnings + ] + primary_artifact_names = set(manifest.artifacts) + manifest, recorded_release_kind = merge_regional_release_manifest( + country, manifest, regional_manifest, + release_kind=regional_release_kind, regional_repo_type=regional_uri_parts["repo_type"], artifact_prefix=regional_artifact_prefix, path_template=regional_path_template, ) + regional_artifacts = { + name: artifact + for name, artifact in manifest.artifacts.items() + if name not in primary_artifact_names + } compatibility_basis, warnings = validate_release_manifest( manifest, model_package, @@ -919,6 +1197,7 @@ def certify( core_version=core_version, qualified_pair_id=qualified_pair_id, ) + warnings = warnings + regional_warnings for filename in required_supplemental_release_files( country, manifest, uri_parts @@ -934,6 +1213,12 @@ def certify( raise CertificationError( f"Certified dataset artifact is not reachable: {default_artifact.uri}" ) + if check_artifacts and recorded_release_kind is not None: + for name, artifact in regional_artifacts.items(): + if not head_artifact(artifact, token=token): + raise CertificationError( + f"Regional artifact {name!r} is not reachable: {artifact.uri}" + ) model_wheel = fetch_pypi_wheel_metadata(model_package, model_version) country_payload = build_country_manifest_payload( @@ -970,6 +1255,7 @@ def certify( uri_parts=uri_parts, regional_manifest_uri=regional_manifest_uri, regional_uri_parts=regional_uri_parts, + regional_release_kind=recorded_release_kind, ), manifest_sha256, warnings, @@ -1011,6 +1297,8 @@ def certify_data_release( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", + drop_regional_release: bool = False, core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, ) -> CertificationResult: @@ -1027,6 +1315,25 @@ def certify_data_release( model_package = COUNTRY_MODEL_PACKAGES[country] model_version = model_version or installed_model_version(model_package) strategy = certification_strategy(country, data_producer) + bundle_path = bundle_path or default_bundle_source_path() + bundle = json.loads(bundle_path.read_text()) + + # Certification rewrites data_releases. wholesale, so a run + # without the regional manifest would silently drop the regional routing. + current_regional_uri = ((bundle.get("data_releases") or {}).get(country) or {}).get( + "regional_source_manifest_uri" + ) + if ( + current_regional_uri is not None + and regional_manifest_uri is None + and not drop_regional_release + ): + raise CertificationError( + f"The bundle's {country.upper()} data release routes regions to " + f"{current_regional_uri}. Pass --regional-manifest-uri (that release " + "or its replacement) to keep regional routing, or " + "--drop-regional-release to remove it." + ) data_release, manifest_sha256, warnings = strategy.certify( country=country, @@ -1038,11 +1345,10 @@ def certify_data_release( regional_manifest_uri=regional_manifest_uri, regional_artifact_prefix=regional_artifact_prefix, regional_path_template=regional_path_template, + regional_release_kind=regional_release_kind, core_version=core_version, qualified_pair_id=qualified_pair_id, ) - bundle_path = bundle_path or default_bundle_source_path() - bundle = json.loads(bundle_path.read_text()) bundle.setdefault("data_releases", {})[country] = data_release bundle.setdefault("countries", {}).setdefault(country, {})["model_package"] = ( model_package diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 05d062fea..6ae786d65 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -47,11 +47,22 @@ class BuiltWithModelPackage(PackageVersion): class DataBuildInfo(BaseModel): build_id: Optional[str] = None + cut_tag: Optional[str] = None + """The immutable tag of one cut of a release line. Microcosm keeps + ``build_id`` equal to the line name for every cut, so the cut tag is what + tells two cuts apart (see ``release_build_id``).""" built_at: Optional[str] = None built_with_core_package: Optional[PackageVersion] = None built_with_model_package: Optional[BuiltWithModelPackage] = None +def release_build_id(build: Optional[DataBuildInfo]) -> Optional[str]: + """Identify one data build: its cut tag when it has one, else its build id.""" + if build is None: + return None + return build.cut_tag or build.build_id + + class ArtifactPathReference(BaseModel): path: str revision: Optional[str] = None @@ -175,6 +186,9 @@ class CertifiedDataArtifact(BaseModel): build_id: Optional[str] = None +RegionalReleaseKind = Literal["local_area", "area_coded_national"] + + class CountryReleaseManifest(BaseModel): schema_version: int = 1 bundle_id: Optional[str] = None @@ -186,6 +200,11 @@ class CountryReleaseManifest(BaseModel): default_dataset: str datasets: dict[str, ArtifactPathReference] = Field(default_factory=dict) region_datasets: dict[str, ArtifactPathTemplate] = Field(default_factory=dict) + regional_release_kind: Optional[RegionalReleaseKind] = None + """How certification routed the local region types, when it did: + ``local_area`` for a non-default local-area release, and + ``area_coded_national`` for another release's national dataset that + carries area codes (and its area weight matrices).""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) @@ -529,11 +548,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="exact_build_model_version", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=built_with_model.version, built_with_model_git_sha=built_with_model.git_sha, data_build_fingerprint=built_with_model.data_build_fingerprint, @@ -548,11 +563,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="matching_data_build_fingerprint", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=built_with_model.version, built_with_model_git_sha=built_with_model.git_sha, data_build_fingerprint=built_with_model.data_build_fingerprint, @@ -568,11 +579,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="legacy_compatible_model_package", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=( built_with_model.version if built_with_model is not None else None ), diff --git a/src/policyengine/provenance/uk_uc_floor.py b/src/policyengine/provenance/uk_uc_floor.py new file mode 100644 index 000000000..2fe476b32 --- /dev/null +++ b/src/policyengine/provenance/uk_uc_floor.py @@ -0,0 +1,159 @@ +"""Universal Credit coverage floor for UK certification candidates. + +In August 2026 (#491) the UK default was reverted after a certified dataset +carried about half of Universal Credit's caseload and spend: its calibration +had no benefit targets, and no check compared its outputs with administrative +totals. This is the standing floor #491 proposed. A candidate dataset's +benefit units with Universal Credit, and its Universal Credit spend, in Great +Britain must each be within 10% of DWP's and the OBR's figures for the same +year. + +Both benchmarks cover Great Britain (Northern Ireland's Universal Credit is +administered separately), so Northern Ireland households are excluded from +the measurement. The receipt holds weighted national totals only. + +Run it with ``scripts/check_uk_uc_floor.py``. +""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Optional + +import numpy as np +from pydantic import BaseModel + +UC_FLOOR_RESOURCE = ("data", "qualification", "uk_uc_floor.json") +NORTHERN_IRELAND_REGION = "NORTHERN_IRELAND" + + +class UCFloorBenchmark(BaseModel): + value: float + source: str + url: str + + +class UCFloorYear(BaseModel): + benefit_units: UCFloorBenchmark + spend: UCFloorBenchmark + + +class UCFloorBenchmarks(BaseModel): + schema_version: int + description: str + tolerance: float + geography: str + years: dict[int, UCFloorYear] + + +class UCFloorMeasure(BaseModel): + name: str + measured: float + benchmark: float + ratio: float + passed: bool + source: str + + +class UCFloorReceipt(BaseModel): + schema_version: int = 1 + check: str = "uk_uc_floor" + dataset: str + year: int + geography: str + tolerance: float + model_version: Optional[str] = None + measures: list[UCFloorMeasure] + passed: bool + + +def load_uc_floor_benchmarks() -> UCFloorBenchmarks: + resource = files("policyengine").joinpath(*UC_FLOOR_RESOURCE) + return UCFloorBenchmarks.model_validate(json.loads(resource.read_text())) + + +def measure_great_britain_uc(simulation, year: int) -> tuple[float, float]: + """Return weighted (benefit units with UC, UC spend) in Great Britain.""" + universal_credit = np.asarray( + simulation.calculate("universal_credit", year), dtype=float + ) + weights = np.asarray(simulation.calculate("benunit_weight", year), dtype=float) + northern_ireland_household = ( + np.asarray(simulation.calculate("region", year)).astype(str) + == NORTHERN_IRELAND_REGION + ).astype(float) + # Mapping to benefit units spreads a household's value over its members; + # any positive share marks a benefit unit in a Northern Ireland household. + northern_ireland = ( + np.asarray( + simulation.map_result(northern_ireland_household, "household", "benunit"), + dtype=float, + ) + > 0 + ) + great_britain = ~northern_ireland + benefit_units = float(weights[great_britain & (universal_credit > 0)].sum()) + spend = float((universal_credit * weights)[great_britain].sum()) + return benefit_units, spend + + +def evaluate_uc_floor( + *, + benefit_units: float, + spend: float, + year: int, + dataset: str, + model_version: Optional[str] = None, + benchmarks: Optional[UCFloorBenchmarks] = None, +) -> UCFloorReceipt: + benchmarks = benchmarks or load_uc_floor_benchmarks() + if year not in benchmarks.years: + raise ValueError( + f"No Universal Credit floor benchmarks for {year}; " + f"available years: {sorted(benchmarks.years)}" + ) + year_benchmarks = benchmarks.years[year] + measures = [] + for name, measured, benchmark in ( + ("benefit_units", benefit_units, year_benchmarks.benefit_units), + ("spend", spend, year_benchmarks.spend), + ): + ratio = measured / benchmark.value + measures.append( + UCFloorMeasure( + name=name, + measured=measured, + benchmark=benchmark.value, + ratio=ratio, + passed=abs(ratio - 1) <= benchmarks.tolerance, + source=benchmark.source, + ) + ) + return UCFloorReceipt( + dataset=dataset, + year=year, + geography=benchmarks.geography, + tolerance=benchmarks.tolerance, + model_version=model_version, + measures=measures, + passed=all(measure.passed for measure in measures), + ) + + +def check_uc_floor(dataset_path: str, year: int, *, dataset: str) -> UCFloorReceipt: + """Measure a UK dataset file and evaluate it against the floor.""" + from importlib.metadata import version + + from policyengine_uk import Microsimulation + + benefit_units, spend = measure_great_britain_uc( + Microsimulation(dataset=dataset_path), year + ) + return evaluate_uc_floor( + benefit_units=benefit_units, + spend=spend, + year=year, + dataset=dataset, + model_version=version("policyengine-uk"), + ) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index b18a77f0f..1a148ee5a 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -14,6 +14,9 @@ CertificationError, build_country_manifest_payload, certify_data_release, + merge_area_coded_national_release_manifest, + merge_local_area_release_manifest, + merge_regional_release_manifest, merge_us_local_area_release_manifest, merge_us_regional_release_manifest, merge_us_state_release_manifest, @@ -22,7 +25,10 @@ should_validate_vendored_artifacts, validate_release_manifest, ) -from policyengine.provenance.manifest import DataReleaseManifest +from policyengine.provenance.manifest import ( + CountryReleaseManifest, + DataReleaseManifest, +) TAG = "populace-us-2024-aaaaaaa-20260101" MANIFEST_URI = ( @@ -171,6 +177,13 @@ def _local_area_release_manifest_payload() -> dict: "dataset_role": "non_default_local_area", "is_default": False, "default_datasets": {}, + "build": { + "build_id": US_LOCAL_AREA_TAG, + "built_with_model_package": { + "name": "policyengine-us", + "version": "1.723.0", + }, + }, "artifacts": { "populace_us_2024_acs_local": { "kind": "microdata", @@ -222,6 +235,12 @@ def _uk_release_manifest_payload() -> dict: return payload +def _tmp_bundle(tmp_path: Path) -> Path: + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text(json.dumps(_bundle_source_payload()) + "\n") + return bundle_path + + def _bundle_source_payload() -> dict: return { "schema_version": 2, @@ -697,6 +716,7 @@ def test__given_us_local_area_manifest__then_certifies_shared_regional_dataset( assert ( release["datasets"]["populace_us_2024_acs_local"]["repo_type"] == "dataset" ) + assert release["regional_release_kind"] == "local_area" assert result.dataset_count == 5 def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsupported( @@ -790,6 +810,7 @@ def test__given_missing_populace_us_source_coverage__then_raises(self, tmp_path) data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) def test__given_unreachable_artifact__then_raises(self, tmp_path): @@ -821,6 +842,7 @@ def test__given_unreachable_artifact__then_raises(self, tmp_path): data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) def test__given_unreachable_vendored_artifact__then_raises(self, tmp_path): @@ -856,6 +878,7 @@ def test__given_unreachable_vendored_artifact__then_raises(self, tmp_path): data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) @@ -969,3 +992,473 @@ def test__given_microcosm_producer__then_still_drops_area_region_templates(self) assert self._payload()["region_datasets"] == { "national": {"path_template": "populace_us_2024.h5"} } + + +UK_NATIONAL_LINE = "microcosm-uk-2024-25-national" +UK_NATIONAL_TAG = f"{UK_NATIONAL_LINE}-20261101T000000Z-aaaaaaaa" +UK_NATIONAL_URI = ( + "hf://dataset/policyengine/populace-uk-private" + f"@{UK_NATIONAL_TAG}/releases/{UK_NATIONAL_LINE}/release_manifest.json" +) +UK_DENSE_LINE = "microcosm-uk-2024-25-dense" +UK_DENSE_TAG = f"{UK_DENSE_LINE}-20261101T000000Z-bbbbbbbb" +UK_DENSE_URI = ( + "hf://dataset/policyengine/populace-uk-private" + f"@{UK_DENSE_TAG}/releases/{UK_DENSE_LINE}/release_manifest.json" +) +EFRS_URI = ( + "hf://model/policyengine/policyengine-uk-data-private" + "@0aa3eee1cdae8f8125f4aff126a8f01957595dc3/releases/1.58.0/release_manifest.json" +) +UK_MODEL_VERSION = "2.126.12" +EFRS_REPO = "policyengine/policyengine-uk-data-private" + + +def _microcosm_uk_national_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "build": { + "build_id": UK_NATIONAL_LINE, + "cut_tag": UK_NATIONAL_TAG, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + }, + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_NATIONAL_TAG, + "sha256": "a" * 64, + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_NATIONAL_TAG, + }, + }, + } + + +def _microcosm_uk_dense_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "build": { + "build_id": UK_DENSE_LINE, + "cut_tag": UK_DENSE_TAG, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + }, + "artifacts": { + "microcosm_uk_2024_25_dense": { + "kind": "microdata", + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_DENSE_TAG, + "sha256": "b" * 64, + }, + "rowwise_candidate_manifest": { + "kind": "diagnostics", + "path": "rowwise_candidate_manifest.json", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_DENSE_TAG, + }, + }, + } + + +def _efrs_release_payload() -> dict: + """Shaped like policyengine-uk-data's enhanced FRS 1.58.0 release.""" + + def artifact(kind: str, path: str, digit: str) -> dict: + return { + "kind": kind, + "path": path, + "repo_id": EFRS_REPO, + "revision": "1.58.0", + "sha256": digit * 64, + } + + return { + "schema_version": 1, + "data_package": {"name": "policyengine-uk-data", "version": "1.58.0"}, + "default_datasets": { + "national": "enhanced_frs_2024_25", + "baseline": "frs_2024_25", + }, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": "==2.122.2"}, + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"}, + ], + "build": { + "build_id": "policyengine-uk-data-1.58.0", + "built_with_model_package": { + "name": "policyengine-uk", + "version": "2.122.2", + }, + }, + "artifacts": { + "enhanced_frs_2024_25": artifact( + "microdata", "enhanced_frs_2024_25.h5", "1" + ), + "frs_2024_25": artifact("microdata", "frs_2024_25.h5", "2"), + "local_authority_weights": artifact( + "weights", "local_authority_weights.h5", "3" + ), + "local_geography_weights": artifact( + "weights", "local_geography_weights.csv.gz", "4" + ), + "parliamentary_constituency_weights": artifact( + "weights", "parliamentary_constituency_weights.h5", "5" + ), + }, + } + + +def _response(payload: dict) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.content = json.dumps(payload).encode() + return response + + +def _certify_uk(bundle_path: Path, responses: list, **kwargs): + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=responses, + ), + patch( + "policyengine.provenance.certification.head_artifact", + return_value=True, + ), + patch( + "policyengine.provenance.certification.head_artifact_reference", + return_value=True, + ), + patch( + "policyengine.provenance.certification.fetch_pypi_wheel_metadata", + return_value={"sha256": "d" * 64, "url": "https://example"}, + ), + patch( + "policyengine.provenance.certification.policyengine_version", + return_value="9.9.9", + ), + ): + return certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=UK_NATIONAL_URI, + model_version=UK_MODEL_VERSION, + bundle_path=bundle_path, + **kwargs, + ) + + +def _sim_api_routes_both_area_types(release: dict) -> bool: + # policyengine-sim-api's uk_area_regions_routed: a UK default without area + # codes is accepted only when both area region types are routed. + manifest = CountryReleaseManifest.model_validate(release) + return all( + region_type in manifest.region_datasets + for region_type in ("constituency", "local_authority") + ) + + +class TestUKRegionalMerge: + def _national(self) -> DataReleaseManifest: + return DataReleaseManifest.model_validate(_microcosm_uk_national_payload()) + + def test__given_uk_local_area_release__then_routes_constituencies_and_las(self): + merged, kind = merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_microcosm_uk_dense_payload()), + regional_repo_type="dataset", + ) + + assert kind == "local_area" + assert merged.metadata["region_datasets"] == { + "national": {"path_template": "microcosm_uk_2024_25.h5"}, + "constituency": {"path_template": "microcosm_uk_2024_25_dense.h5"}, + "local_authority": {"path_template": "microcosm_uk_2024_25_dense.h5"}, + } + dense = merged.artifacts["microcosm_uk_2024_25_dense"] + assert dense.revision == UK_DENSE_TAG + assert dense.repo_type == "dataset" + # Only the microdata joins; the local release's evidence stays there. + assert "rowwise_candidate_manifest" not in merged.artifacts + + def test__given_efrs_as_area_coded_national__then_carries_dataset_and_matrices( + self, + ): + merged, kind = merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_efrs_release_payload()), + release_kind="area_coded_national", + regional_repo_type="model", + ) + + assert kind == "area_coded_national" + assert merged.default_datasets == {"national": "microcosm_uk_2024_25"} + region_datasets = merged.metadata["region_datasets"] + assert region_datasets["constituency"] == { + "path_template": "enhanced_frs_2024_25.h5" + } + assert region_datasets["local_authority"] == region_datasets["constituency"] + carried = { + "enhanced_frs_2024_25", + "local_authority_weights", + "local_geography_weights", + "parliamentary_constituency_weights", + } + assert carried <= set(merged.artifacts) + assert "frs_2024_25" not in merged.artifacts + for name in carried: + artifact = merged.artifacts[name] + assert (artifact.repo_id, artifact.revision) == (EFRS_REPO, "1.58.0") + assert artifact.repo_type == "model" + + def test__given_national_release_without_kind__then_names_the_flag(self): + with pytest.raises( + CertificationError, match="--regional-release-kind area_coded_national" + ): + merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_efrs_release_payload()), + ) + + def test__given_area_coded_release_without_a_matrix__then_rejects_it(self): + payload = _efrs_release_payload() + payload["artifacts"].pop("local_authority_weights") + + with pytest.raises(CertificationError, match="local_authority_weights.h5"): + merge_area_coded_national_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(payload), + ) + + def test__given_local_area_release_as_area_coded_national__then_rejects_it(self): + with pytest.raises(CertificationError, match="must be a national release"): + merge_area_coded_national_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_microcosm_uk_dense_payload()), + ) + + def test__given_us_area_coded_national__then_rejects_it(self): + with pytest.raises(CertificationError, match="only supported for the UK"): + merge_area_coded_national_release_manifest( + "us", + DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ), + DataReleaseManifest.model_validate(_efrs_release_payload()), + ) + + def test__given_uk_local_area_release__then_generic_merge_checks_role(self): + payload = _microcosm_uk_dense_payload() + payload["is_default"] = True + + with pytest.raises( + CertificationError, match="UK local-area release must declare is_default" + ): + merge_local_area_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(payload), + ) + + +class TestUKRegionalCertification: + def test__given_national_first_efrs_route__then_bundle_routes_areas_to_efrs( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + + result = _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_efrs_release_payload()), + ], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["default_dataset"] == "microcosm_uk_2024_25" + assert release["regional_release_kind"] == "area_coded_national" + assert release["regional_source_manifest_uri"] == EFRS_URI + assert release["region_datasets"]["constituency"] == { + "path_template": "enhanced_frs_2024_25.h5" + } + assert _sim_api_routes_both_area_types(release) + # sim-api binds a matrix only to the release of the dataset it + # reweights: the same repo and revision. + datasets = release["datasets"] + for name in ( + "enhanced_frs_2024_25", + "parliamentary_constituency_weights", + "local_authority_weights", + ): + assert (datasets[name]["repo_id"], datasets[name]["revision"]) == ( + EFRS_REPO, + "1.58.0", + ) + assert datasets[name]["repo_type"] == "model" + # Diagnostics stay under the release directory of the national cut. + assert datasets["calibration_diagnostics"]["path"] == ( + f"releases/{UK_NATIONAL_LINE}/calibration_diagnostics.json" + ) + # The cut tag, not the line name, identifies the certified build. + assert release["version"] == UK_NATIONAL_TAG + assert release["certification"]["data_build_id"] == UK_NATIONAL_TAG + assert result.build_id == UK_NATIONAL_TAG + + def test__given_ship_together_dense_route__then_bundle_routes_areas_to_local( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["regional_release_kind"] == "local_area" + assert release["region_datasets"]["local_authority"] == { + "path_template": "microcosm_uk_2024_25_dense.h5" + } + assert release["datasets"]["microcosm_uk_2024_25_dense"]["revision"] == ( + UK_DENSE_TAG + ) + assert _sim_api_routes_both_area_types(release) + + def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): + efrs = _efrs_release_payload() + efrs["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.122.2"} + ] + + with pytest.raises(CertificationError, match="Regional release .*2.126.12"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(efrs), + ], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + ) + + def test__given_unreachable_regional_artifact__then_refuses(self, tmp_path): + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + ), + patch( + "policyengine.provenance.certification.head_artifact", + side_effect=lambda artifact, token=None: ( + artifact.path != "microcosm_uk_2024_25_dense.h5" + ), + ), + pytest.raises(CertificationError, match="Regional artifact"), + ): + certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=UK_NATIONAL_URI, + regional_manifest_uri=UK_DENSE_URI, + model_version=UK_MODEL_VERSION, + bundle_path=_tmp_bundle(tmp_path), + ) + + def test__given_routed_bundle__then_recertifying_without_regional_refuses( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + with pytest.raises(CertificationError, match="--drop-regional-release"): + _certify_uk(bundle_path, [_response(_microcosm_uk_national_payload())]) + + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert "regional_source_manifest_uri" not in release + assert "regional_release_kind" not in release + assert set(release["region_datasets"]) == {"national"} + + def test__given_bundle_wrapper_regional_kind__then_forwards_to_certifier( + self, monkeypatch + ): + bundle_script = _load_bundle_script(monkeypatch) + captured: dict[str, list[str]] = {} + fake = ModuleType("certify_data_release") + + def fake_main(argv): + captured["argv"] = argv + return 0 + + fake.main = fake_main + monkeypatch.setitem(sys.modules, "certify_data_release", fake) + + assert ( + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--regional-manifest-uri", + EFRS_URI, + "--regional-release-kind", + "area_coded_national", + "--drop-regional-release", + ] + ) + == 0 + ) + argv = captured["argv"] + assert argv[argv.index("--regional-release-kind") + 1] == ( + "area_coded_national" + ) + assert "--drop-regional-release" in argv diff --git a/tests/test_constituency_impact.py b/tests/test_constituency_impact.py index d7b9b29c5..59820857b 100644 --- a/tests/test_constituency_impact.py +++ b/tests/test_constituency_impact.py @@ -6,6 +6,7 @@ import pytest from microdf import MicroDataFrame +from policyengine.data.uk_geography_assets import packaged_uk_area_list_path from policyengine.outputs.constituency_impact import ( ConstituencyImpact, compute_uk_constituency_impacts, @@ -155,31 +156,29 @@ def test_relative_change(tmp_path): ) -def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( +def test_compute_labels_with_packaged_area_list_by_default( tmp_path, ): - """The helper can enrich labels from local CSV without matrix assets.""" + """Labels come from the packaged list, not local or downloaded files.""" baseline = _make_sim( { - "constituency_code_oa": ["C001", "C002"], + "constituency_code_oa": ["E14001063", "UNLISTED"], "household_net_income": [100.0, 200.0], "household_weight": [1.0, 1.0], } ) reform = _make_sim( { - "constituency_code_oa": ["C001", "C002"], + "constituency_code_oa": ["E14001063", "UNLISTED"], "household_net_income": [110.0, 220.0], "household_weight": [1.0, 1.0], } ) - csv_path = tmp_path / CONSTITUENCY_ASSET_SPEC.lookup_csv_filename - pd.DataFrame( - [ - {"code": "C001", "name": "A", "x": 0, "y": 0}, - {"code": "C002", "name": "B", "x": 1, "y": 1}, - ] - ).to_csv(csv_path, index=False) + # A stale lookup in a local search dir (the old GCS download cache) is + # ignored. + pd.DataFrame([{"code": "E14001063", "name": "Stale", "x": 0, "y": 0}]).to_csv( + tmp_path / CONSTITUENCY_ASSET_SPEC.lookup_csv_filename, index=False + ) with patch( "policyengine.outputs.uk_geography_impact.default_local_search_dirs", @@ -191,9 +190,13 @@ def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( download_missing_assets=True, ) - assert impact.constituency_csv_path == str(csv_path) - assert len(impact.constituency_results) == 2 - assert impact.constituency_results[0]["constituency_name"] == "A" + assert impact.constituency_csv_path == packaged_uk_area_list_path("constituency") + assert impact.constituency_csv_path.endswith("constituencies_2024.csv") + by_code = {row["constituency_code"]: row for row in impact.constituency_results} + assert by_code["E14001063"]["constituency_name"] == "Aldershot" + assert (by_code["E14001063"]["x"], by_code["E14001063"]["y"]) == (56, -40) + assert by_code["UNLISTED"]["constituency_name"] == "UNLISTED" + assert by_code["UNLISTED"]["x"] is None def test_compute_constituency_impacts_does_not_require_lookup_csv_or_matrix( diff --git a/tests/test_local_authority_impact.py b/tests/test_local_authority_impact.py index 3c5365027..8ecfcf842 100644 --- a/tests/test_local_authority_impact.py +++ b/tests/test_local_authority_impact.py @@ -6,6 +6,7 @@ import pytest from microdf import MicroDataFrame +from policyengine.data.uk_geography_assets import packaged_uk_area_list_path from policyengine.outputs.local_authority_impact import ( LocalAuthorityImpact, compute_uk_local_authority_impacts, @@ -157,31 +158,29 @@ def test_relative_change(tmp_path): ) -def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( +def test_compute_labels_with_packaged_area_list_by_default( tmp_path, ): - """The helper can enrich labels from local CSV without matrix assets.""" + """Labels come from the packaged list, not local or downloaded files.""" baseline = _make_sim( { - "la_code_oa": ["LA001", "LA002"], + "la_code_oa": ["N09000011", "UNLISTED"], "household_net_income": [100.0, 200.0], "household_weight": [1.0, 1.0], } ) reform = _make_sim( { - "la_code_oa": ["LA001", "LA002"], - "household_net_income": [115.0, 230.0], + "la_code_oa": ["N09000011", "UNLISTED"], + "household_net_income": [110.0, 220.0], "household_weight": [1.0, 1.0], } ) - csv_path = tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename - pd.DataFrame( - [ - {"code": "LA001", "name": "A", "x": 0, "y": 0}, - {"code": "LA002", "name": "B", "x": 1, "y": 1}, - ] - ).to_csv(csv_path, index=False) + # A stale lookup in a local search dir (the old GCS download cache) is + # ignored. + pd.DataFrame([{"code": "N09000011", "name": "Stale", "x": 0, "y": 0}]).to_csv( + tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename, index=False + ) with patch( "policyengine.outputs.uk_geography_impact.default_local_search_dirs", @@ -193,9 +192,17 @@ def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( download_missing_assets=True, ) - assert impact.local_authority_csv_path == str(csv_path) - assert len(impact.local_authority_results) == 2 - assert impact.local_authority_results[0]["local_authority_name"] == "A" + assert impact.local_authority_csv_path == packaged_uk_area_list_path( + "local_authority" + ) + assert impact.local_authority_csv_path.endswith("local_authorities_2023.csv") + by_code = { + row["local_authority_code"]: row for row in impact.local_authority_results + } + assert by_code["N09000011"]["local_authority_name"] == "Ards and North Down" + assert (by_code["N09000011"]["x"], by_code["N09000011"]["y"]) == (-3, 16) + assert by_code["UNLISTED"]["local_authority_name"] == "UNLISTED" + assert by_code["UNLISTED"]["x"] is None def test_compute_local_authority_impacts_does_not_require_lookup_csv_or_matrix( diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 132e1599c..2586c593d 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -672,6 +672,41 @@ def test__given_matching_fingerprint__then_certification_allows_reuse(self): assert certification.built_with_model_version == "1.601.0" assert certification.certified_for_model_version == "1.602.0" + def test__given_release_with_cut_tag__then_cut_tag_is_the_data_build_id(self): + # Microcosm keeps build_id equal to the line name for every cut, so + # the runtime check must name the cut the same way certification did. + get_data_release_manifest.cache_clear() + payload = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "build": { + "build_id": "microcosm-uk-2024-25-national", + "cut_tag": "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa", + "built_with_model_package": { + "name": "policyengine-us", + "version": "1.601.0", + }, + }, + "compatible_model_packages": [], + "default_datasets": {"national": "enhanced_cps_2024"}, + "artifacts": {}, + } + + with patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ): + certification = certify_data_release_compatibility( + "us", + runtime_model_version="1.601.0", + ) + + assert certification.compatibility_basis == "exact_build_model_version" + assert certification.data_build_id == ( + "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + ) + get_data_release_manifest.cache_clear() + def test__given_private_manifest_unavailable__then_bundled_certification_is_used( self, ): diff --git a/tests/test_uk_area_lists.py b/tests/test_uk_area_lists.py new file mode 100644 index 000000000..62d17fee8 --- /dev/null +++ b/tests/test_uk_area_lists.py @@ -0,0 +1,114 @@ +"""Tests for the UK area lists shipped with the package.""" + +import csv +import re +from collections import Counter +from unittest.mock import patch + +import pytest + +from policyengine.countries.uk.regions import build_uk_region_registry +from policyengine.data.uk_geography_assets import ( + LOCAL_AUTHORITY_ASSET_SPEC, + UK_AREA_LIST_FILES, + packaged_uk_area_list_path, +) + +GSS_CODE = re.compile(r"^[A-Z]\d{8}$") + +# The 17 districts abolished on 1 April 2023, when Cumberland, Westmorland and +# Furness, North Yorkshire and Somerset became unitary authorities. +ABOLISHED_2023_DISTRICTS = { + "E07000026", + "E07000027", + "E07000028", + "E07000029", + "E07000030", + "E07000031", + "E07000163", + "E07000164", + "E07000165", + "E07000166", + "E07000167", + "E07000168", + "E07000169", + "E07000187", + "E07000188", + "E07000189", + "E07000246", +} +NEW_2023_UNITARIES = {"E06000063", "E06000064", "E06000065", "E06000066"} + + +def _rows(geography_type: str) -> list[dict]: + with open(packaged_uk_area_list_path(geography_type), encoding="utf-8") as file: + return list(csv.DictReader(file)) + + +@pytest.mark.parametrize("geography_type", sorted(UK_AREA_LIST_FILES)) +def test_area_list_rows_are_well_formed(geography_type): + rows = _rows(geography_type) + codes = [row["code"] for row in rows] + + assert len(codes) == len(set(codes)) + assert all(GSS_CODE.match(code) for code in codes) + assert all(row["name"].strip() and row["name"] != row["code"] for row in rows) + hexes = [(int(row["x"]), int(row["y"])) for row in rows] + assert len(hexes) == len(set(hexes)), "two areas share a hex" + + +def test_local_authority_list_is_the_april_2023_frame(): + codes = {row["code"] for row in _rows("local_authority")} + + assert len(codes) == 361 + assert Counter(code[0] for code in codes) == {"E": 296, "W": 22, "S": 32, "N": 11} + assert "N09000011" in codes # Ards and North Down + assert NEW_2023_UNITARIES <= codes + assert not ABOLISHED_2023_DISTRICTS & codes + + +def test_constituency_list_is_the_2024_frame_including_northern_ireland(): + codes = [row["code"] for row in _rows("constituency")] + + assert len(codes) == 650 + assert Counter(code[:3] for code in codes) == { + "E14": 543, + "S14": 57, + "W07": 32, + "N05": 18, + } + + +def test_weight_matrix_lookup_stays_paired_with_its_matrix(): + # The legacy local-authority matrix indexes areas by the row order of the + # CSV it was built with, so the packaged list must not replace it. + assert ( + LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename == "local_authorities_2021.csv" + ) + assert UK_AREA_LIST_FILES["local_authority"] != ( + LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename + ) + + +def test_unknown_geography_type_has_no_packaged_list(): + with pytest.raises(ValueError, match="No packaged UK area list"): + packaged_uk_area_list_path("ward") + + +def test_registry_builds_every_area_without_network(): + # Before the lists were packaged, these regions came from a GCS download + # that silently returned no areas when it failed. + with patch( + "policyengine_core.tools.google_cloud.download_gcs_file", + side_effect=AssertionError("network access"), + ): + registry = build_uk_region_registry( + include_constituencies=True, + include_local_authorities=True, + ) + + assert len(registry.get_by_type("national")) == 1 + assert len(registry.get_by_type("country")) == 4 + assert len(registry.get_by_type("constituency")) == 650 + assert len(registry.get_by_type("local_authority")) == 361 + assert registry.get("local_authority/N09000011").label == "Ards and North Down" diff --git a/tests/test_uk_regions.py b/tests/test_uk_regions.py index b3697679b..77d250bcb 100644 --- a/tests/test_uk_regions.py +++ b/tests/test_uk_regions.py @@ -17,6 +17,7 @@ build_uk_region_registry, uk_region_registry, ) +from policyengine.provenance.manifest import CountryReleaseManifest class TestUKCountries: @@ -303,6 +304,146 @@ def test__given_local_authorities_included__then_filters_on_dataset_geography( assert local_authority.scoping_strategy.variable_value == "LA001" +MICROCOSM_NATIONAL_TAG = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" +MICROCOSM_DENSE_TAG = "microcosm-uk-2024-25-dense-20261101T000000Z-bbbbbbbb" +MICROCOSM_NATIONAL_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + f"@{MICROCOSM_NATIONAL_TAG}" +) +EFRS_AREA_URI = ( + "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0" +) +MICROCOSM_DENSE_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + f"@{MICROCOSM_DENSE_TAG}" +) + + +def _uk_routed_manifest(kind: str) -> CountryReleaseManifest: + """A certified Microcosm national bundle whose area regions are routed.""" + if kind == "area_coded_national": + area_name, area_reference = ( + "enhanced_frs_2024_25", + { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + }, + ) + else: + area_name, area_reference = ( + "microcosm_uk_2024_25_dense", + { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": MICROCOSM_DENSE_TAG, + }, + ) + area_template = {"path_template": area_reference["path"]} + return CountryReleaseManifest.model_validate( + { + "country_id": "uk", + "policyengine_version": "9.9.9", + "model_package": {"name": "policyengine-uk", "version": "2.126.12"}, + "data_package": { + "name": "microcosm-data", + "version": "0.1.0", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "release_manifest_revision": MICROCOSM_NATIONAL_TAG, + }, + "default_dataset": "microcosm_uk_2024_25", + "datasets": { + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": MICROCOSM_NATIONAL_TAG, + }, + area_name: area_reference, + }, + "region_datasets": { + "national": {"path_template": "microcosm_uk_2024_25.h5"}, + "constituency": area_template, + "local_authority": area_template, + }, + "regional_release_kind": kind, + } + ) + + +class TestUKAreaRegionRouting: + """Area regions load the dataset certification routes them to.""" + + def test__given_unrouted_bundle__then_area_regions_filter_the_national_dataset( + self, + ): + registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True + ) + + assert registry.get("constituency/E14001063").dataset_path is None + assert registry.get("local_authority/N09000011").dataset_path is None + assert len(uk_region_registry.get_by_type("constituency")) == 0 + + def test__given_national_first_bundle__then_area_regions_load_the_efrs( + self, monkeypatch + ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("area_coded_national"), + ) + + default = build_uk_region_registry() + registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True + ) + + # An area-coded national route does not add area regions by default. + assert len(default) == 5 + assert registry.get("uk").dataset_path == MICROCOSM_NATIONAL_URI + assert registry.get("country/wales").dataset_path is None + constituency = registry.get("constituency/E14001063") + assert constituency.dataset_path == EFRS_AREA_URI + assert constituency.scoping_strategy.variable_name == "constituency_code_oa" + local_authority = registry.get("local_authority/N09000011") + assert local_authority.dataset_path == EFRS_AREA_URI + assert local_authority.scoping_strategy.variable_name == "la_code_oa" + + def test__given_local_area_bundle__then_default_registry_adds_area_regions( + self, monkeypatch + ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("local_area"), + ) + + registry = build_uk_region_registry() + + assert len(registry.get_by_type("constituency")) == 650 + assert len(registry.get_by_type("local_authority")) == 361 + assert registry.get("constituency/N05000001").dataset_path == ( + MICROCOSM_DENSE_URI + ) + assert registry.get("local_authority/E06000063").dataset_path == ( + MICROCOSM_DENSE_URI + ) + assert registry.get("uk").dataset_path == MICROCOSM_NATIONAL_URI + + def test__given_local_area_bundle__then_explicit_exclusion_wins(self, monkeypatch): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("local_area"), + ) + + registry = build_uk_region_registry( + include_constituencies=False, include_local_authorities=False + ) + + assert len(registry) == 5 + + # The pinned UK model's Region enum contains nine English ITL1 regions and # Scotland, Wales and Northern Ireland. Country is derived, not a stored input. _REGION_COUNTRIES = [ diff --git a/tests/test_uk_uc_floor.py b/tests/test_uk_uc_floor.py new file mode 100644 index 000000000..461749344 --- /dev/null +++ b/tests/test_uk_uc_floor.py @@ -0,0 +1,84 @@ +"""Tests for the UK Universal Credit coverage floor.""" + +import pytest + +from policyengine.provenance.uk_uc_floor import ( + evaluate_uc_floor, + load_uc_floor_benchmarks, + measure_great_britain_uc, +) + + +def test_benchmarks_cite_a_source_for_every_measure(): + benchmarks = load_uc_floor_benchmarks() + + assert benchmarks.tolerance == 0.1 + assert benchmarks.geography == "great_britain" + assert 2025 in benchmarks.years + for year in benchmarks.years.values(): + for benchmark in (year.benefit_units, year.spend): + assert benchmark.value > 0 + assert benchmark.source + assert benchmark.url.startswith("https://") + + +def test_floor_passes_within_tolerance_and_fails_outside_it(): + year = load_uc_floor_benchmarks().years[2025] + + passing = evaluate_uc_floor( + benefit_units=year.benefit_units.value * 0.95, + spend=year.spend.value * 1.09, + year=2025, + dataset="candidate", + ) + failing = evaluate_uc_floor( + benefit_units=year.benefit_units.value * 0.5, + spend=year.spend.value, + year=2025, + dataset="candidate", + ) + + assert passing.passed + assert not failing.passed + measures = {measure.name: measure for measure in failing.measures} + assert not measures["benefit_units"].passed + assert measures["benefit_units"].ratio == pytest.approx(0.5) + assert measures["spend"].passed + + +def test_floor_refuses_a_year_without_benchmarks(): + with pytest.raises(ValueError, match="No Universal Credit floor benchmarks"): + evaluate_uc_floor(benefit_units=1, spend=1, year=2019, dataset="candidate") + + +def test_measurement_excludes_northern_ireland(): + from policyengine_uk import Simulation + + simulation = Simulation( + situation={ + "people": { + "a": {"age": {2025: 30}}, + "b": {"age": {2025: 40}}, + "c": {"age": {2025: 50}}, + }, + "benunits": { + "ni_one": {"members": ["a"]}, + "ni_two": {"members": ["b"]}, + "london": {"members": ["c"]}, + }, + "households": { + "belfast": { + "members": ["a", "b"], + "region": {2025: "NORTHERN_IRELAND"}, + }, + "london": {"members": ["c"], "region": {2025: "LONDON"}}, + }, + } + ) + universal_credit = simulation.calculate("universal_credit", 2025) + assert (universal_credit > 0).all() + + benefit_units, spend = measure_great_britain_uc(simulation, 2025) + + assert benefit_units == pytest.approx(1) + assert spend == pytest.approx(float(universal_credit[2])) From 0d17238676f133f133d45201532b91828b959e2c Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 15:59:00 +0100 Subject: [PATCH 2/7] Name builds by cut tag in qualification and check a regional release's core Follows #574 on main. Its qualified-pair runtime check compared the bundle's data_build_id, which certification now takes from the cut tag, with the release manifest's build_id, the line name for every Microcosm cut: a qualified cut would be refused at import, and another cut of the same line would have matched. The runtime check, the UK candidate pair's data build id and the Zenodo description now use release_build_id. With --core-version, a regional release must also cover the pinned core by its own build or claims; a qualified pair names the primary release only. Refs #553 Co-Authored-By: Claude Opus 5.5 --- changelog.d/553.fixed.md | 2 +- docs/engineering/skills/data-certification.md | 6 ++- src/policyengine/provenance/certification.py | 16 +++++++- src/policyengine/provenance/manifest.py | 7 +--- .../provenance/uk_qualification.py | 7 ++-- src/policyengine/provenance/zenodo.py | 5 ++- tests/test_certify_data_release.py | 37 +++++++++++++++++++ tests/test_qualified_pair_certification.py | 22 +++++++++++ 8 files changed, 86 insertions(+), 16 deletions(-) diff --git a/changelog.d/553.fixed.md b/changelog.d/553.fixed.md index 64cac1129..3b5d15e5c 100644 --- a/changelog.d/553.fixed.md +++ b/changelog.d/553.fixed.md @@ -1 +1 @@ -Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose). +Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version (and core, with `--core-version`) by its own claims, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose). diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index cbd06576d..c3f3477d7 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -79,8 +79,10 @@ Microcosm national release with one of: claim first. Its `frs_2024_25` is not carried, so keep it loadable with a `dataset_overlays.uk` entry. -Either way, certification checks the regional release's own model -compatibility and HEAD-checks every artifact it carries. +Either way, certification checks that the regional release serves the pinned +model, and the pinned core when `--core-version` is given, by its own build or +compatibility claims (a `--qualified-pair-id` names the primary release only), +and HEAD-checks every artifact it carries. ### Re-certifying a routed bundle diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 9594b4f02..04d1cd7d1 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -1162,12 +1162,17 @@ def certify( regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token ) - # The regional release serves the same model, so it must make its - # own compatibility claim for the pinned version. + # The regional release runs on the same model and core, so it must + # cover the pinned versions by its own build or claims. A qualified + # pair names the primary release only, so it does not extend here. try: _, regional_warnings = model_compatibility_basis( regional_manifest, model_package, model_version ) + if core_version is not None: + _check_core_compatibility( + regional_manifest, core_version, None, regional_warnings + ) except CertificationError as exc: raise CertificationError( f"Regional release {regional_manifest_uri}: {exc}" @@ -1309,6 +1314,13 @@ def certify_data_release( core and data release when neither the build-time model nor a publisher claim covers them; the certificate then names the pair, whose promotable qualification receipt must be in the ledger. + + ``regional_manifest_uri`` routes the country's local region types to a + regional release, read as ``regional_release_kind``. That release must + serve the pinned model, and the pinned core when ``core_version`` is + given, by its own build or claims: a qualified pair names the primary + release only. While the bundle routes regions, a run without + ``regional_manifest_uri`` is refused unless ``drop_regional_release``. """ if country not in COUNTRY_MODEL_PACKAGES: raise CertificationError(f"Unknown country {country!r}.") diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 6ae786d65..d43e3f74b 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -493,11 +493,8 @@ def certify_data_release_compatibility( # exact-pair certificate beyond the model, core and recorded data build # that qualification tested. Check them before any acceptance path returns. bundled_certification = country_manifest.certification - remote_build_id = ( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ) + # Named as certification names it: the cut tag when the release has one. + remote_build_id = release_build_id(data_release_manifest.build) if ( bundled_certification is not None and bundled_certification.compatibility_basis == "qualified_exact_pair" diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index 5e98532d3..d9c54d8c5 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -48,7 +48,7 @@ from packaging.specifiers import SpecifierSet from packaging.version import InvalidVersion, Version -from policyengine.provenance.manifest import DataReleaseManifest +from policyengine.provenance.manifest import DataReleaseManifest, release_build_id from policyengine.provenance.qualification import ( REQUIRED_GATES, CandidatePair, @@ -286,9 +286,8 @@ def resolve_candidate( artifact = manifest.artifacts[dataset] if not artifact.sha256 or not artifact.size_bytes: raise ValueError(f"uk-data {data_version} {dataset} has no sha256 or size.") - build_id = ( - manifest.build.build_id if manifest.build and manifest.build.build_id else None - ) + # Named as certification names it: the cut tag when the release has one. + build_id = release_build_id(manifest.build) if build_id is None: raise ValueError(f"uk-data {data_version} records no build id.") diff --git a/src/policyengine/provenance/zenodo.py b/src/policyengine/provenance/zenodo.py index 8fe4c41b9..aa7b68579 100644 --- a/src/policyengine/provenance/zenodo.py +++ b/src/policyengine/provenance/zenodo.py @@ -37,6 +37,7 @@ PreservationMirror, get_data_release_manifest, get_release_manifest, + release_build_id, ) from .trace import canonical_json_bytes @@ -94,8 +95,8 @@ def deposition_metadata( f"{country_id}-{country_manifest.policyengine_version}" ) build_id = None - if data_release_manifest is not None and data_release_manifest.build is not None: - build_id = data_release_manifest.build.build_id + if data_release_manifest is not None: + build_id = release_build_id(data_release_manifest.build) certified = country_manifest.certified_data_artifact description_parts = [ f"Certification record for the PolicyEngine {country_id} certified " diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 1a148ee5a..dd0e5c64f 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1374,6 +1374,43 @@ def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): regional_release_kind="area_coded_national", ) + def test__given_pinned_core__then_the_regional_release_must_cover_it( + self, tmp_path + ): + # A qualified pair names the primary release only, so the regional + # release covers the pinned core by its own build or claims. + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.0"} + ] + + with pytest.raises( + CertificationError, match="Regional release .*policyengine-core 3.33.0" + ): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(_efrs_release_payload())], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + core_version="3.33.0", + ) + + efrs = _efrs_release_payload() + efrs["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.0"} + ] + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [_response(national), _response(efrs)], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + core_version="3.33.0", + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["certification"]["certified_for_core_version"] == "3.33.0" + assert release["regional_release_kind"] == "area_coded_national" + def test__given_unreachable_regional_artifact__then_refuses(self, tmp_path): with ( patch( diff --git a/tests/test_qualified_pair_certification.py b/tests/test_qualified_pair_certification.py index 953dd192c..e2297230f 100644 --- a/tests/test_qualified_pair_certification.py +++ b/tests/test_qualified_pair_certification.py @@ -278,6 +278,28 @@ def test_online_other_data_build_is_refused(self): remote=_uk_manifest(), ) + def test_online_exact_pair_on_a_cut_is_named_by_its_cut_tag(self): + """A Microcosm cut keeps its line name as ``build.build_id``. + + Certification names the build by ``build.cut_tag`` (#553), so the + runtime check must too: the certified cut passes, and another cut of + the same line, which shares the line name, is refused. + """ + line = "microcosm-uk-2024-25-national" + certified_cut = f"{line}-20261101T000000Z-aaaaaaaa" + remote = _uk_manifest() + remote.build.build_id = line + remote.build.cut_tag = certified_cut + + certification = self._certify( + _qualified(build_id=certified_cut), "2.126.8", remote=remote + ) + assert certification.compatibility_basis == "qualified_exact_pair" + + remote.build.cut_tag = f"{line}-20261102T000000Z-bbbbbbbb" + with pytest.raises(ValueError, match="not certified.*data build"): + self._certify(_qualified(build_id=certified_cut), "2.126.8", remote=remote) + def test_online_other_core_is_refused(self): """Interrupted review of #574: an exact pair was honoured whatever core ran.""" From 0f0d5cdb654481c9159f577bfd6552f9d9242fb3 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Mon, 19 Oct 2026 10:10:39 +0100 Subject: [PATCH 3/7] Say which bundle entries lack regional_release_kind The US entry on main routes states and districts to the ACS-local release but was certified before the field existed, so it has none until its next certification records local_area. Refs #553 Co-Authored-By: Claude Opus 5.5 --- docs/release-bundles.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 632846eef..927c77d3b 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -253,7 +253,9 @@ routed those region types: `local_area` for a non-default local-area release, and `area_coded_national` (UK only) for another release's national dataset whose households carry area codes, certified together with that release's area weight matrices at their own repo and revision. It is absent when nothing -is routed, or for a legacy US per-state release. +is routed, for a legacy US per-state release, and in an entry certified before +the field existed (as the US ACS-local routing is until its next +certification). Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's From 247b90d351a1012c81f0f26c14f6f2aa3d860b36 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 14:28:09 +0100 Subject: [PATCH 4/7] Share one model-basis decision and derive the regional release kinds validate_release_manifest and the regional check now take the built-with or publisher-claim basis, its warning and the refusal reason from one helper, so the two cannot drift; the primary path adds its core and qualified-pair handling around it. REGIONAL_RELEASE_KINDS comes from the RegionalReleaseKind literal, certify_data_release.py takes its choices from it, and scripts/bundle.py forwards the flag without its own copy. Refs #553 Co-Authored-By: Claude Opus 5.5 --- scripts/bundle.py | 7 +- scripts/certify_data_release.py | 3 +- src/policyengine/provenance/certification.py | 85 ++++++++++---------- tests/test_certify_data_release.py | 35 ++++++++ 4 files changed, 81 insertions(+), 49 deletions(-) diff --git a/scripts/bundle.py b/scripts/bundle.py index 3efb69077..9f8d47126 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -211,11 +211,10 @@ def _parser() -> argparse.ArgumentParser: ) certify.add_argument( "--regional-release-kind", - choices=["local_area", "area_coded_national"], help=( - "How to read the regional release. Defaults to local_area; " - "area_coded_national (UK only) routes area regions to another " - "national release's area-coded dataset and weight matrices." + "How to read the regional release; forwarded to " + "scripts/certify_data_release.py, which lists and checks the kinds. " + "Defaults to local_area." ), ) certify.add_argument( diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index ad7cef764..538aa61bb 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -35,6 +35,7 @@ from generate_bundle_artifacts import generate # noqa: E402 from policyengine.provenance.certification import ( # noqa: E402 + REGIONAL_RELEASE_KINDS, certify_data_release, ) @@ -66,7 +67,7 @@ def main(argv=None) -> int: parser.add_argument( "--regional-release-kind", default="local_area", - choices=["local_area", "area_coded_national"], + choices=REGIONAL_RELEASE_KINDS, help=( "local_area: a non-default local-area release. " "area_coded_national (UK only): another certified national release " diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 04d1cd7d1..97341c75f 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -44,7 +44,7 @@ import warnings from dataclasses import dataclass, field from pathlib import Path -from typing import Literal, Optional +from typing import Literal, Optional, get_args import requests @@ -101,10 +101,7 @@ "us": ("state", "congressional_district"), "uk": ("constituency", "local_authority"), } -REGIONAL_RELEASE_KINDS: tuple[RegionalReleaseKind, ...] = ( - "local_area", - "area_coded_national", -) +REGIONAL_RELEASE_KINDS: tuple[RegionalReleaseKind, ...] = get_args(RegionalReleaseKind) # A UK area-coded national release (the enhanced FRS) serves constituency and # local-authority runs with these weight matrices as well as its households. # Consumers bind a matrix only to the release of the dataset it reweights, so @@ -302,7 +299,7 @@ def validate_release_manifest( if not artifact.sha256: warnings.append(f"artifact {name!r} has no sha256") - built_with, claim_specifiers, claim_matches = _model_claims( + basis, basis_warning, refusal = _release_model_basis( manifest, model_package, model_version ) @@ -321,16 +318,10 @@ def validate_release_manifest( ) return BASIS_QUALIFIED_PAIR, warnings - if built_with == model_version: - return BASIS_BUILT_WITH, warnings - if claim_matches: - warnings.append( - f"certifying {model_package} {model_version} on the publisher's " - f"compatibility claim {claim_specifiers}; the data was built " - f"with {built_with or 'an unrecorded model version'} — the test " - "suite is the arbiter" - ) - return BASIS_PUBLISHER_CLAIM, warnings + if basis is not None: + if basis_warning is not None: + warnings.append(basis_warning) + return basis, warnings if qualified_pair_id is not None: if core_version is None: raise CertificationError( @@ -344,38 +335,53 @@ def validate_release_manifest( ) return BASIS_QUALIFIED_PAIR, warnings raise CertificationError( - f"{model_package} {model_version} matches neither the build-time " - f"model ({built_with or 'unrecorded'}) nor any publisher " - f"compatibility claim {claim_specifiers or '(none declared)'}; " - "a new data build, a published compatibility claim or an automated " - "qualification of the exact pair is required." + f"{refusal}; a new data build, a published compatibility claim or an " + "automated qualification of the exact pair is required." ) -def _model_claims( +def _release_model_basis( manifest: DataReleaseManifest, model_package: str, model_version: str, -) -> tuple[Optional[str], list[str], bool]: - """A release's build-time model version and its claims on ``model_version``. +) -> tuple[Optional[str], Optional[str], str]: + """How a release serves ``model_version`` by its own build or claims. - Returns ``(built_with, claim_specifiers, claim_matches)``. + Returns ``(basis, warning, refusal)``. ``basis`` is the built-with or + publisher-claim basis, or ``None`` when neither covers the version; + ``warning`` goes with the publisher-claim basis; ``refusal`` says why + neither applies, for the caller to complete with its remedy. """ built_with = None if manifest.build is not None and manifest.build.built_with_model_package: build_pkg = manifest.build.built_with_model_package if build_pkg.name == model_package: built_with = build_pkg.version - claim_specifiers = [ package.specifier for package in manifest.compatible_model_packages if package.name == model_package ] - claim_matches = any( + if built_with == model_version: + return BASIS_BUILT_WITH, None, "" + if any( _specifier_matches(model_version, specifier) for specifier in claim_specifiers + ): + return ( + BASIS_PUBLISHER_CLAIM, + f"certifying {model_package} {model_version} on the publisher's " + f"compatibility claim {claim_specifiers}; the data was built " + f"with {built_with or 'an unrecorded model version'} — the test " + "suite is the arbiter", + "", + ) + return ( + None, + None, + f"{model_package} {model_version} matches neither the build-time " + f"model ({built_with or 'unrecorded'}) nor any publisher " + f"compatibility claim {claim_specifiers or '(none declared)'}", ) - return built_with, claim_specifiers, claim_matches def model_compatibility_basis( @@ -390,24 +396,15 @@ def model_compatibility_basis( uses this for a regional release, which no qualification names. Returns ``(compatibility_basis, warnings)``. """ - built_with, claim_specifiers, claim_matches = _model_claims( + basis, warning, refusal = _release_model_basis( manifest, model_package, model_version ) - if built_with == model_version: - return BASIS_BUILT_WITH, [] - if claim_matches: - return BASIS_PUBLISHER_CLAIM, [ - f"certifying {model_package} {model_version} on the publisher's " - f"compatibility claim {claim_specifiers}; the data was built " - f"with {built_with or 'an unrecorded model version'} — the test " - "suite is the arbiter" - ] - raise CertificationError( - f"{model_package} {model_version} matches neither the build-time " - f"model ({built_with or 'unrecorded'}) nor any publisher " - f"compatibility claim {claim_specifiers or '(none declared)'}; " - "a new data build or a published compatibility claim is required." - ) + if basis is None: + raise CertificationError( + f"{refusal}; a new data build or a published compatibility claim " + "is required." + ) + return basis, [] if warning is None else [warning] def _check_core_compatibility( diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index dd0e5c64f..8e390f0c8 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -20,6 +20,7 @@ merge_us_local_area_release_manifest, merge_us_regional_release_manifest, merge_us_state_release_manifest, + model_compatibility_basis, parse_manifest_uri, required_supplemental_release_files, should_validate_vendored_artifacts, @@ -274,6 +275,40 @@ def test__given_bare_path__then_raises(self): parse_manifest_uri("https://example.com/manifest.json") +class TestModelBasisAgreement: + """The primary and regional checks share one model-basis decision.""" + + def test__given_claim_only__then_both_paths_give_the_same_warning(self): + manifest = _manifest() + manifest.build.built_with_model_package = None + + basis, warnings = validate_release_manifest( + manifest, "policyengine-us", "1.723.0" + ) + regional_basis, regional_warnings = model_compatibility_basis( + manifest, "policyengine-us", "1.723.0" + ) + + assert basis == regional_basis == "legacy_compatible_model_package" + assert regional_warnings == [ + warning for warning in warnings if "compatibility claim" in warning + ] + + def test__given_no_basis__then_both_refusals_state_the_same_reason(self): + manifest = _manifest() + manifest.build.built_with_model_package = None + + with pytest.raises(CertificationError) as primary: + validate_release_manifest(manifest, "policyengine-us", "9.9.9") + with pytest.raises(CertificationError) as regional: + model_compatibility_basis(manifest, "policyengine-us", "9.9.9") + + reason = str(regional.value).split(";")[0] + assert str(primary.value).startswith(reason + ";") + assert "automated qualification" in str(primary.value) + assert "automated qualification" not in str(regional.value) + + class TestValidateReleaseManifest: def test__given_built_with_match__then_built_with_basis_no_warnings(self): basis, warnings = validate_release_manifest( From 9d10eb79bb4bf1e956b44e37f10b02ac516a2c07 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 14:29:57 +0100 Subject: [PATCH 5/7] Type the regional merge with its artifact and country types The new merge helpers take a CertifiedCountry literal ("us" or "uk"), which certify_data_release narrows from its string argument once, with the same "Unknown country" error, so mypy catches a wrong country before the region-type lookup raises at run time. _add_regional_artifact takes a DataReleaseArtifact, and the strategy keeps its regional artifacts as dict[str, DataReleaseArtifact]. Refs #553 Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/certification.py | 44 ++++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 97341c75f..c9c7fefe8 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -44,7 +44,7 @@ import warnings from dataclasses import dataclass, field from pathlib import Path -from typing import Literal, Optional, get_args +from typing import Literal, Optional, cast, get_args import requests @@ -54,6 +54,7 @@ ) from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, + DataReleaseArtifact, DataReleaseManifest, RegionalReleaseKind, _specifier_matches, @@ -66,7 +67,8 @@ r"/(?P.+)$" ) -COUNTRY_MODEL_PACKAGES = { +CertifiedCountry = Literal["us", "uk"] +COUNTRY_MODEL_PACKAGES: dict[CertifiedCountry, str] = { "us": "policyengine-us", "uk": "policyengine-uk", } @@ -97,7 +99,7 @@ } # Region types a regional release serves. Their regions load the routed dataset # and keep row-filtering it on their own geography column. -LOCAL_AREA_REGION_TYPES = { +LOCAL_AREA_REGION_TYPES: dict[CertifiedCountry, tuple[str, ...]] = { "us": ("state", "congressional_district"), "uk": ("constituency", "local_authority"), } @@ -633,7 +635,14 @@ def merge_us_state_release_manifest( return DataReleaseManifest.model_validate(primary_payload) -def _local_area_region_types(country: str) -> tuple[str, ...]: +def _certified_country(country: str) -> CertifiedCountry: + """``country`` as a certified country id; any other value is refused.""" + if country not in get_args(CertifiedCountry): + raise CertificationError(f"Unknown country {country!r}.") + return cast(CertifiedCountry, country) + + +def _local_area_region_types(country: CertifiedCountry) -> tuple[str, ...]: try: return LOCAL_AREA_REGION_TYPES[country] except KeyError: @@ -643,7 +652,7 @@ def _local_area_region_types(country: str) -> tuple[str, ...]: def _route_region_types( - country: str, + country: CertifiedCountry, primary_manifest: DataReleaseManifest, primary_payload: dict, path: str, @@ -670,9 +679,9 @@ def _route_region_types( def _add_regional_artifact( - merged_artifacts: dict, + merged_artifacts: dict[str, dict], name: str, - artifact, + artifact: DataReleaseArtifact, regional_repo_type: Optional[Literal["model", "dataset"]], ) -> None: if name in merged_artifacts: @@ -690,7 +699,7 @@ def _add_regional_artifact( def merge_local_area_release_manifest( - country: str, + country: CertifiedCountry, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, @@ -764,7 +773,7 @@ def merge_us_local_area_release_manifest( def merge_area_coded_national_release_manifest( - country: str, + country: CertifiedCountry, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, @@ -835,7 +844,7 @@ def merge_area_coded_national_release_manifest( def merge_regional_release_manifest( - country: str, + country: CertifiedCountry, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, @@ -1088,7 +1097,7 @@ class DataProducerCertificationStrategy: def certify( self, *, - country: str, + country: CertifiedCountry, manifest_uri: str, model_package: str, model_version: str, @@ -1110,7 +1119,7 @@ class LegacyDataProducerCertificationStrategy(DataProducerCertificationStrategy) def certify( self, *, - country: str, + country: CertifiedCountry, manifest_uri: str, model_package: str, model_version: str, @@ -1135,7 +1144,7 @@ class PopulaceDataProducerCertificationStrategy(DataProducerCertificationStrateg def certify( self, *, - country: str, + country: CertifiedCountry, manifest_uri: str, model_package: str, model_version: str, @@ -1153,7 +1162,7 @@ def certify( ) regional_uri_parts = None recorded_release_kind = None - regional_artifacts: dict = {} + regional_artifacts: dict[str, DataReleaseArtifact] = {} regional_warnings: list[str] = [] if regional_manifest_uri is not None: regional_manifest, _, regional_uri_parts = fetch_release_manifest( @@ -1319,9 +1328,8 @@ def certify_data_release( release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. """ - if country not in COUNTRY_MODEL_PACKAGES: - raise CertificationError(f"Unknown country {country!r}.") - model_package = COUNTRY_MODEL_PACKAGES[country] + certified_country = _certified_country(country) + model_package = COUNTRY_MODEL_PACKAGES[certified_country] model_version = model_version or installed_model_version(model_package) strategy = certification_strategy(country, data_producer) bundle_path = bundle_path or default_bundle_source_path() @@ -1345,7 +1353,7 @@ def certify_data_release( ) data_release, manifest_sha256, warnings = strategy.certify( - country=country, + country=certified_country, manifest_uri=manifest_uri, model_package=model_package, model_version=model_version, From 8eba4d32d31141203c8eb9ec989e6a2839c4950d Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 14:32:02 +0100 Subject: [PATCH 6/7] Refuse dropping and re-routing regions in one run; test cut tags certify-data with both --drop-regional-release and --regional-manifest-uri kept the routing without a word, because the drop flag was read only without a regional release. The combination is now refused before any fetch. New tests set build.cut_tag in a release manifest and check that zenodo.deposition_metadata and uk_qualification.resolve_candidate name the build by it, as certification and the runtime check already did. Refs #553 Co-Authored-By: Claude Opus 5.5 --- changelog.d/553.fixed.md | 2 +- docs/engineering/skills/data-certification.md | 2 +- scripts/bundle.py | 5 ++++- scripts/certify_data_release.py | 5 ++++- src/policyengine/provenance/certification.py | 5 +++++ tests/test_certify_data_release.py | 20 +++++++++++++++++++ tests/test_uk_qualification_pipeline.py | 16 +++++++++++++++ tests/test_zenodo_mirror.py | 14 +++++++++++++ 8 files changed, 65 insertions(+), 4 deletions(-) diff --git a/changelog.d/553.fixed.md b/changelog.d/553.fixed.md index 3b5d15e5c..d9706db56 100644 --- a/changelog.d/553.fixed.md +++ b/changelog.d/553.fixed.md @@ -1 +1 @@ -Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version (and core, with `--core-version`) by its own claims, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose). +Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version (and core, with `--core-version`) by its own claims, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose, and cannot be combined with `--regional-manifest-uri`). diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index c3f3477d7..42eec89be 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -90,7 +90,7 @@ Certification rewrites `data_releases.` from the manifests it is given. While the bundle routes regions (`regional_source_manifest_uri` is set), a run without `--regional-manifest-uri` is refused; pass the current regional release again (or its replacement), or `--drop-regional-release` to remove the -routing on purpose. +routing on purpose. The two flags cannot be combined. ### US diff --git a/scripts/bundle.py b/scripts/bundle.py index 9f8d47126..42179f4fe 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -220,7 +220,10 @@ def _parser() -> argparse.ArgumentParser: certify.add_argument( "--drop-regional-release", action="store_true", - help="Remove the bundle's current regional routing on re-certification.", + help=( + "Remove the bundle's current regional routing on re-certification. " + "Cannot be combined with --regional-manifest-uri." + ), ) certify.add_argument( "--regional-artifact-prefix", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 538aa61bb..f3a720ee4 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -79,7 +79,10 @@ def main(argv=None) -> int: parser.add_argument( "--drop-regional-release", action="store_true", - help="Remove the bundle's current regional routing on re-certification.", + help=( + "Remove the bundle's current regional routing on re-certification. " + "Cannot be combined with --regional-manifest-uri." + ), ) parser.add_argument( "--regional-artifact-prefix", diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index c9c7fefe8..31327f576 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -1329,6 +1329,11 @@ def certify_data_release( ``regional_manifest_uri`` is refused unless ``drop_regional_release``. """ certified_country = _certified_country(country) + if drop_regional_release and regional_manifest_uri is not None: + raise CertificationError( + "--drop-regional-release removes the bundle's regional routing, so it " + "cannot be combined with --regional-manifest-uri." + ) model_package = COUNTRY_MODEL_PACKAGES[certified_country] model_version = model_version or installed_model_version(model_package) strategy = certification_strategy(country, data_producer) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 8e390f0c8..74964a50f 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1498,6 +1498,26 @@ def test__given_routed_bundle__then_recertifying_without_regional_refuses( assert "regional_release_kind" not in release assert set(release["region_datasets"]) == {"national"} + def test__given_drop_and_a_regional_release__then_refuses_both(self, tmp_path): + # Dropping the routing and routing to a release are opposite requests; + # neither silently wins. + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=AssertionError("no fetch"), + ), + pytest.raises(CertificationError, match="cannot be combined"), + ): + certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=UK_NATIONAL_URI, + regional_manifest_uri=UK_DENSE_URI, + drop_regional_release=True, + model_version=UK_MODEL_VERSION, + bundle_path=_tmp_bundle(tmp_path), + ) + def test__given_bundle_wrapper_regional_kind__then_forwards_to_certifier( self, monkeypatch ): diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 5372f77ae..ca10bc9ff 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -233,6 +233,22 @@ def test_latest_data_release_ignores_non_version_tags(): assert latest_data_release(["1.57.4", "1.58.0", "v1.11.1", "version_a"]) == "1.58.0" +def test_resolution_names_the_data_build_by_its_cut_tag(): + """A release that records a cut tag is named by it, as certification + names it, so the ledger's data build id matches the certificate's.""" + + cut = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + manifest = { + **MANIFEST, + "build": {"build_id": "microcosm-uk-2024-25-national", "cut_tag": cut}, + } + manifest_bytes = json.dumps(manifest).encode() + + resolved, _ = _resolver(hf_file=lambda repo, rev, path: manifest_bytes) + + assert resolved.pair.data.build_id == cut + + def test_resolution_refuses_unpublished_versions(): with pytest.raises(ValueError): _resolver(model_version="9.9.9") diff --git a/tests/test_zenodo_mirror.py b/tests/test_zenodo_mirror.py index a894ee7d0..28a275058 100644 --- a/tests/test_zenodo_mirror.py +++ b/tests/test_zenodo_mirror.py @@ -404,6 +404,20 @@ def test__given_manifests__then_version_and_description_pinned(self): description = metadata["description"] assert "populace-us-2024" in description + def test__given_a_cut_tag__then_the_description_names_the_cut(self): + # Microcosm keeps build_id equal to the line name on every cut, so the + # deposit names the cut, as certification does. + cut = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + manifest = _us_data_release_manifest() + manifest.build.build_id = "microcosm-uk-2024-25-national" + manifest.build.cut_tag = cut + + description = deposition_metadata(get_release_manifest("us"), manifest)[ + "description" + ] + + assert f"Data build: {cut}." in description + class TestCLI: def test__given_cli__then_prints_mirrors_json(self, monkeypatch, capsys): From 521ab47806b674ce6d35c6ef1eb18b41cfe08c4e Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 14:34:36 +0100 Subject: [PATCH 7/7] Compare qualification candidates only within main's data line Automated UK qualification ordered pairs by (model, data release, core) version across data producers. With a Microcosm certificate on main (microcosm-data 0.1.0), every policyengine-uk release paired with a uk-data release (1.58.x) compared as newer, so the workflow would qualify it and promotion would ready a PR switching the default back. A candidate's data line is its data package and repository. A pair from another line than main's certificate now gets the action other_line, unless forced, and is left out of promotion: it is neither readied nor closed. Changing the line stays a reviewed certification. Pairs on main's line behave as before. Refs #553 Co-Authored-By: Claude Opus 5.5 --- changelog.d/553.fixed.md | 2 +- .../provenance/uk_qualification.py | 55 +++++++++++++++++-- tests/test_uk_qualification_pipeline.py | 46 ++++++++++++++++ 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/changelog.d/553.fixed.md b/changelog.d/553.fixed.md index d9706db56..6609f6a22 100644 --- a/changelog.d/553.fixed.md +++ b/changelog.d/553.fixed.md @@ -1 +1 @@ -Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version (and core, with `--core-version`) by its own claims, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose, and cannot be combined with `--regional-manifest-uri`). +Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version (and core, with `--core-version`) by its own claims, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose, and cannot be combined with `--regional-manifest-uri`). Automated UK qualification no longer orders a candidate from another data line against main's certificate: with a Microcosm certificate on main, every uk-data pair compared as newer and would have been promoted back. Such a pair is now skipped (`other_line`) unless forced, and promotion never readies or closes it. diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index d9c54d8c5..cb1c9988c 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -7,8 +7,10 @@ Find the candidate pair: the newest (or requested) policyengine-uk wheel from PyPI, the core it needs, and the newest (or requested) uk-data release at the data repository's current manifest revision. Decide - whether to qualify it, or whether it is already certified, stale, or a - duplicate of an attempt in the ledger. + whether to qualify it, or whether it is already certified, stale, a + duplicate of an attempt in the ledger, or from another data line than + main's certificate (``other_line``: versions are compared only within + one line, so a uk-data pair is never ordered against a Microcosm one). ``prepare`` On a checkout of ``main``, certify the pair into the bundle manifest (exact ``==`` claim backed by the pair id when no publisher claim covers @@ -126,7 +128,7 @@ def certification_environment(environ: Mapping[str, str]) -> dict[str, str]: REPO_ROOT = Path(__file__).resolve().parents[3] -Action = Literal["qualify", "already_certified", "stale", "duplicate"] +Action = Literal["qualify", "already_certified", "stale", "duplicate", "other_line"] # --- Resolution ------------------------------------------------------------ @@ -331,6 +333,32 @@ def resolve_candidate( # --- Deciding what to do ------------------------------------------------------ +def certified_data_line(bundle: Mapping[str, Any]) -> tuple[str, str]: + """The data package and repository of main's UK certificate. + + Promotion orders pairs by their data release version, which compares + only within one data line: a uk-data release number says nothing about + a Microcosm build. A pair from another line is never older or newer + than main's certificate; changing the line is a reviewed certification. + """ + + data_package = bundle["data_releases"]["uk"]["data_package"] + return (data_package["name"], data_package["repo_id"]) + + +def data_line(pair: CandidatePair) -> tuple[str, str]: + """The data package and repository a candidate pair's data comes from.""" + + return (pair.data.data_package, pair.data.repo_id) + + +def _same_line_receipts( + receipts: Iterable[QualificationReceipt], bundle: Mapping[str, Any] +) -> list[QualificationReceipt]: + line = certified_data_line(bundle) + return [receipt for receipt in receipts if data_line(receipt.pair) == line] + + def certified_order_key(bundle: Mapping[str, Any]) -> tuple[str, str, str]: """(model version, data release, core version) certified on a branch.""" @@ -391,6 +419,19 @@ def candidate_action( promotion never waits on a manual ``--force``. """ + line = certified_data_line(bundle) + if data_line(pair) != line: + if force: + return "qualify", ( + f"forced across data lines: main certifies {line}, the pair " + f"is from {data_line(pair)}" + ) + return "other_line", ( + f"main certifies {line[0]} from {line[1]}; the pair is from " + f"{data_line(pair)[0]} in {data_line(pair)[1]}. Automated " + "qualification compares versions only within one data line, and " + "changing the line is a reviewed certification" + ) current = certified_order_key(bundle) record = bundle["data_releases"]["uk"] if pair.order_key == current and ( @@ -526,7 +567,9 @@ def requalification_target( ) -> Optional[CandidatePair]: """The promotion target, when it must be requalified on current main.""" - target = select_promotion_target(list(receipts), certified_order_key(bundle)) + target = select_promotion_target( + _same_line_receipts(receipts, bundle), certified_order_key(bundle) + ) if target is None: return None pair_dir = pair_directory(ledger_root, "uk", target.pair_id) @@ -1523,7 +1566,9 @@ def plan_promotion( A branch that moved since is left alone and reported. """ - receipts = list(receipts) + # Only pairs on main's data line are ordered against it; others are left + # alone (neither promoted nor closed). + receipts = _same_line_receipts(receipts, bundle) current = certified_order_key(bundle) target = select_promotion_target(receipts, current) baseline_matches = target is not None and ( diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index ca10bc9ff..5d94bf5b4 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -916,6 +916,52 @@ def test_promotion_plan(): assert candidate_branch(rejected.pair) not in plan.close_branches +def _microcosm_certified_bundle(): + bundle = json.loads(json.dumps(BUNDLE)) + bundle["data_releases"]["uk"]["data_package"].update( + name="microcosm-data", + repo_id="policyengine/populace-uk-private", + version="0.1.0", + ) + return bundle + + +def test_a_pair_from_another_data_line_is_never_ordered_against_main(tmp_path): + """A uk-data release number says nothing about a Microcosm build. With a + Microcosm certificate on main, every uk-data pair compared as newer + (1.58.x > 0.1.0) and would have been qualified and promoted back. Now + automated qualification skips it unless forced, and promotion neither + readies nor closes it.""" + + from policyengine.provenance.uk_qualification import requalification_target + + bundle = _microcosm_certified_bundle() + uk_data = make_pair(model="2.900.0", data="1.58.2", core="3.99.0") + + action, reason = candidate_action(uk_data, bundle, tmp_path, base_commit="a" * 40) + assert action == "other_line" + assert "microcosm-data" in reason and "policyengine-uk-data" in reason + assert ( + candidate_action(uk_data, bundle, tmp_path, base_commit="a" * 40, force=True)[0] + == "qualify" + ) + + promotable = _promotion_receipt(pair=uk_data) + branch = candidate_branch(uk_data) + plan = plan_promotion([promotable], bundle, {branch: promotable.harness.git_commit}) + assert plan.target is None and plan.ready_branch is None + assert branch not in plan.close_branches + assert requalification_target([promotable], bundle, tmp_path, "b" * 40) is None + + # On main's own data line the same pair still advances. + assert ( + plan_promotion( + [promotable], BUNDLE, {branch: promotable.harness.git_commit} + ).target + == promotable + ) + + def _promotable_pair_and_branch(core=None): current = BUNDLE["data_releases"]["uk"]["data_package"]["version"] core = core or BUNDLE["packages"]["policyengine-core"]["version"]