From 5b5fbde020d5051adea8a8bd94db7b293aea3f65 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:59:23 +0400 Subject: [PATCH 1/5] Accept UK area records without x and y Make x and y optional on ConstituencyImpactRecord and LocalAuthorityImpactRecord. Both default to None and are left out of every dump while None (pydantic's field-level exclude_if), so a record without them validates and its JSON has no x or y keys rather than "x": null. Apps deployed for older policyengine.py versions are never rebuilt and keep sending both; their records still validate and keep the values. The gateway's /jobs/{job_id} route already drops null fields (response_model_exclude_none), and the new endpoint tests check the JSON it returns for a breakdown result and a national result whose records carry no x or y, and for records that send them as null. Raise the contract's pydantic floor to 2.12, the first release with exclude_if, and relock the contract and the four packages whose locks record its metadata; only that specifier line changes in each lock. Regenerate the gateway's golden OpenAPI, which no longer lists x and y as required. Written red-green: the contract, gateway and generated-client tests failed first because x and y were required ("Field required", and a 500 from the gateway), and because a null x or y was written to the dump. Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 1 + .../pyproject.toml | 2 +- .../macro_output.py | 21 +++-- .../tests/test_local_area_breakdown.py | 42 ++++++++++ .../tests/test_macro_output_uk_geography.py | 84 +++++++++++++++++++ libs/policyengine-simulation-contract/uv.lock | 2 +- libs/policyengine-stage12-persistence/uv.lock | 2 +- .../tests/simulation/test_calculate.py | 18 ++++ .../policyengine-simulation-entry/uv.lock | 2 +- .../policyengine-simulation-executor/uv.lock | 2 +- .../tests/golden/openapi.json | 6 +- .../tests/test_endpoints.py | 66 +++++++++++++++ .../policyengine-simulation-gateway/uv.lock | 2 +- 13 files changed, 232 insertions(+), 18 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 01cc2da84..e7a929695 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -15,6 +15,7 @@ - Ran Stage 12 US national simulations as 20 region-group calls each, and built an independent, immutable Stage 12 cache of three annual US datasets and 60 current-law baselines before each deployment, publishing it only after a recorded determinism check and recording its release in the v2 version manifest under a new key, while the existing key keeps a copy that earlier readers accept - Made the budget refuse a household count outside its country's plausible range (above 200 million in the US or 60 million in the UK, negative or not finite), raising ImplausibleEntityCountError instead of publishing a count like #767's changed: + - Made x and y, each UK constituency and local-authority record's integer map position, optional in the contract; a record now validates without them and its JSON leaves them out instead of writing null, while records from apps deployed for older policyengine.py versions, which still send both, validate and keep them; the gateway's OpenAPI no longer lists them as required, and the contract now requires pydantic 2.12 or later, the first release with field-level exclude_if - Removed the UK constituency and local-authority weight-matrix region fallback (_build_uk_weight_replacement_region, _require_certified_uk_weight_matrix and its Stage 12 call); each deployment rebuilds only the app for the current policyengine.py version, now 6.10.0, which routes those regions to the local-area dataset, and apps deployed for older versions are not rebuilt and keep their own code; on a bundle that does not route them, an area region is now an unsupported region - Made UK-wide and nation-level runs on a policyengine.py bundle that routes UK area regions to a local-area release dataset (6.10.0) load only the national dataset and return null for constituency_impact and local_authority_impact, whose results come from a breakdown=local_areas run instead; on a bundle that does not route them, those runs still compute both from the national dataset, and Stage 12 leaves the same breakdowns out of the same reports - Enabled every ruff rule (select ALL), ignoring only COM812 and ISC001, which conflict with ruff format, and the D203 and D213 halves of the mutually exclusive docstring pairs; complex functions, long parameter lists and magic numbers were split into smaller helpers and grouped parameters, each behind new characterization tests, across the contract and persistence validation, entry settings, app factory and Stage 12 adapter, gateway route resolution, Stage 12 coordination, workers, parity and qualification, the precompute and national reduce, the routing publisher and its command line, the release sentinel, the package update helper and the shared rules tool; Modal function, FastAPI route, Pydantic validator and command-line signatures and every OpenAPI document are unchanged; the contract library's batch-state builder and SPM selection resolver take their route values as one grouped argument, the simulation observability initializer drops a service_version parameter no caller passed, and no PLR0913 suppression is needed diff --git a/libs/policyengine-simulation-contract/pyproject.toml b/libs/policyengine-simulation-contract/pyproject.toml index 7214eee3b..1de678f91 100644 --- a/libs/policyengine-simulation-contract/pyproject.toml +++ b/libs/policyengine-simulation-contract/pyproject.toml @@ -8,7 +8,7 @@ authors = [ license = {file = "../../LICENSE"} requires-python = ">=3.13" dependencies = [ - "pydantic>=2.0", + "pydantic>=2.12", "packaging>=24", # gateway_models embeds the TelemetryEnvelope from the observability lib. "policyengine-simulation-observability", diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py index 674992ace..1fbece187 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py @@ -4,7 +4,7 @@ from typing import Generic, Literal, TypeVar -from pydantic import BaseModel, ConfigDict, RootModel +from pydantic import BaseModel, ConfigDict, Field, RootModel from policyengine_simulation_contract.spm import SPMComparisonProvenance @@ -237,6 +237,10 @@ class CliffImpactOutput(MacroOutputModel): UKGeographyDisclosureTier = Literal["reported", "approximate"] +def _is_none(value: object) -> bool: + return value is None + + class UKAreaDisclosureFields(MacroOutputModel): """Disclosure tier and 90% intervals of a published UK area's income changes. @@ -264,14 +268,17 @@ class ConstituencyImpactRecord(UKAreaDisclosureFields): ``average_household_income_change`` is the average change per household and ``relative_household_income_change`` the relative change in the area's total household net income, both rounded to the order of their standard error. - ``population`` is the weighted number of households. ``x`` and ``y`` are - integer map-rendering coordinates. + ``population`` is the weighted number of households. + + ``x`` and ``y`` are the area's integer map position. Results from apps + deployed for older policyengine.py versions still carry them; current + results leave both out rather than sending them as null. """ constituency_code: str constituency_name: str - x: int | None - y: int | None + x: int | None = Field(default=None, exclude_if=_is_none) + y: int | None = Field(default=None, exclude_if=_is_none) average_household_income_change: float relative_household_income_change: float population: float @@ -285,8 +292,8 @@ class LocalAuthorityImpactRecord(UKAreaDisclosureFields): local_authority_code: str local_authority_name: str - x: int | None - y: int | None + x: int | None = Field(default=None, exclude_if=_is_none) + y: int | None = Field(default=None, exclude_if=_is_none) average_household_income_change: float relative_household_income_change: float population: float diff --git a/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py b/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py index cdd80eaf1..11cda0173 100644 --- a/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py +++ b/libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py @@ -201,3 +201,45 @@ def test_each_area_level_reads_its_records_as_its_own_type(): assert [type(record) for record in output.local_authority_impact] == [ LocalAuthorityImpactRecord ] + + +def _without_map_position(record: dict[str, object]) -> dict[str, object]: + return {key: value for key, value in record.items() if key not in {"x", "y"}} + + +# Newer executors leave out each area's map position (x and y). +BREAKDOWN_WITHOUT_MAP_POSITIONS = { + **BREAKDOWN, + "constituency_impact": [_without_map_position(CONSTITUENCY)], + "local_authority_impact": [_without_map_position(LOCAL_AUTHORITY)], +} + + +def test_a_breakdown_without_map_positions_round_trips_without_them(): + output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_MAP_POSITIONS) + + assert output.model_dump(mode="json") == BREAKDOWN_WITHOUT_MAP_POSITIONS + + +def test_a_job_status_reads_a_breakdown_without_map_positions(): + response = JobStatusResponse( + status="complete", result=BREAKDOWN_WITHOUT_MAP_POSITIONS + ) + + assert isinstance(response.result, LocalAreaBreakdownOutput) + assert response.model_dump(mode="json")["result"] == ( + BREAKDOWN_WITHOUT_MAP_POSITIONS + ) + + +def test_a_national_result_reads_area_lists_without_map_positions(): + national = { + **NATIONAL, + "constituency_impact": [_without_map_position(CONSTITUENCY)], + "local_authority_impact": [_without_map_position(LOCAL_AUTHORITY)], + } + + response = JobStatusResponse(status="complete", result=national) + + assert isinstance(response.result, SingleYearMacroOutput) + assert response.model_dump(mode="json")["result"] == national diff --git a/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py index 87d8ae48a..9adf5984f 100644 --- a/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py +++ b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + import pytest from policyengine_simulation_contract.macro_output import ( ConstituencyImpactRecord, @@ -72,3 +74,85 @@ def test_published_values_stay_required() -> None: def test_an_empty_area_list_is_valid() -> None: assert GeographicImpactOutput.model_validate([]).root == [] + + +# Executors built for older policyengine.py versions are never rebuilt, and +# they keep sending each area's integer map position as x and y. Newer +# executors send neither. +CONSTITUENCY_WITHOUT_MAP_POSITION = { + **{ + key: value + for key, value in STORED_BEFORE_DISCLOSURE.items() + if key not in {"x", "y"} + }, + **DISCLOSURE_FIELDS, +} +LOCAL_AUTHORITY_WITHOUT_MAP_POSITION = { + "local_authority_code": "E06000063", + "local_authority_name": "Cumberland", + "average_household_income_change": -150.0, + "relative_household_income_change": -0.0026, + "population": 120_000.0, + **DISCLOSURE_FIELDS, +} +AREA_RECORDS_WITHOUT_MAP_POSITION = [ + pytest.param( + ConstituencyImpactRecord, CONSTITUENCY_WITHOUT_MAP_POSITION, id="constituency" + ), + pytest.param( + LocalAuthorityImpactRecord, + LOCAL_AUTHORITY_WITHOUT_MAP_POSITION, + id="local-authority", + ), +] + + +@pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION) +def test_records_without_a_map_position_validate_and_dump_without_one( + record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord], + payload: dict[str, object], +) -> None: + record = record_type.model_validate(payload) + + assert record.x is None + assert record.y is None + assert record.model_dump(mode="python") == payload + assert record.model_dump(mode="json") == payload + assert json.loads(record.model_dump_json()) == payload + + +@pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION) +def test_records_from_older_executors_keep_their_map_position( + record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord], + payload: dict[str, object], +) -> None: + sent_by_older_executor = {**payload, "x": 56, "y": -40} + + record = record_type.model_validate(sent_by_older_executor) + + assert (record.x, record.y) == (56, -40) + assert record.model_dump(mode="json") == sent_by_older_executor + + +@pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION) +def test_a_null_map_position_is_left_out_of_the_dump( + record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord], + payload: dict[str, object], +) -> None: + record = record_type.model_validate({**payload, "x": None, "y": None}) + + assert record.model_dump(mode="json") == payload + assert json.loads(record.model_dump_json()) == payload + + +def test_an_area_list_without_map_positions_dumps_without_them() -> None: + payload = [CONSTITUENCY_WITHOUT_MAP_POSITION, LOCAL_AUTHORITY_WITHOUT_MAP_POSITION] + + output = GeographicImpactOutput.model_validate(payload) + + assert [type(record) for record in output.root] == [ + ConstituencyImpactRecord, + LocalAuthorityImpactRecord, + ] + assert output.model_dump(mode="json") == payload + assert json.loads(output.model_dump_json()) == payload diff --git a/libs/policyengine-simulation-contract/uv.lock b/libs/policyengine-simulation-contract/uv.lock index a1b23fb94..0d52148a2 100644 --- a/libs/policyengine-simulation-contract/uv.lock +++ b/libs/policyengine-simulation-contract/uv.lock @@ -1122,7 +1122,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock index e8d8bbe36..0fc6b2969 100644 --- a/libs/policyengine-stage12-persistence/uv.lock +++ b/libs/policyengine-stage12-persistence/uv.lock @@ -604,7 +604,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py index 105d7bcb6..b4559f1fe 100644 --- a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py +++ b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py @@ -362,6 +362,10 @@ def test_result_assertions_accept_generated_client_objects() -> None: } +def _without_x_and_y(record: Mapping) -> dict: + return {key: value for key, value in record.items() if key not in {"x", "y"}} + + def test_breakdown_assertions_accept_generated_client_objects() -> None: status = JobStatusResponse.from_dict( {"status": "complete", "result": _BREAKDOWN_RESULT} @@ -371,6 +375,20 @@ def test_breakdown_assertions_accept_generated_client_objects() -> None: assert_uk_local_area_breakdown(status.result) +def test_generated_client_reads_area_records_without_x_and_y() -> None: + # Executors built for the current policyengine.py version send no x or y. + breakdown = { + **_BREAKDOWN_RESULT, + "constituency_impact": [_without_x_and_y(_BREAKDOWN_CONSTITUENCY)], + "local_authority_impact": [_without_x_and_y(_BREAKDOWN_LOCAL_AUTHORITY)], + } + + status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown}) + + assert isinstance(status.result, LocalAreaBreakdownOutput) + assert status.result.to_dict() == breakdown + + @pytest.mark.parametrize( ("change", "message"), [ diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index 943b788d5..654495aa8 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -881,7 +881,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index 7453cde31..8469d617e 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1883,7 +1883,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 715cd2c5a..f7a431ce2 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -1140,14 +1140,12 @@ "required": [ "constituency_code", "constituency_name", - "x", - "y", "average_household_income_change", "relative_household_income_change", "population" ], "title": "ConstituencyImpactRecord", - "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income, both rounded to the order of their standard error.\n``population`` is the weighted number of households. ``x`` and ``y`` are\ninteger map-rendering coordinates." + "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income, both rounded to the order of their standard error.\n``population`` is the weighted number of households.\n\n``x`` and ``y`` are the area's integer map position. Results from apps\ndeployed for older policyengine.py versions still carry them; current\nresults leave both out rather than sending them as null." }, "DecileOutput": { "properties": { @@ -1904,8 +1902,6 @@ "required": [ "local_authority_code", "local_authority_name", - "x", - "y", "average_household_income_change", "relative_household_income_change", "population" diff --git a/projects/policyengine-simulation-gateway/tests/test_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_endpoints.py index 72024828f..1c6664f52 100644 --- a/projects/policyengine-simulation-gateway/tests/test_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_endpoints.py @@ -44,6 +44,10 @@ } +def _without_x_and_y(record: dict[str, object]) -> dict[str, object]: + return {key: value for key, value in record.items() if key not in {"x", "y"}} + + def expected_bundle( country: str, model_version: str, @@ -1011,6 +1015,68 @@ def test__given_complete_local_area_breakdown__then_polling_returns_it( assert response.json()["status"] == "complete" assert response.json()["result"] == LOCAL_AREA_BREAKDOWN_RESULT + @pytest.mark.parametrize( + "map_position", + [ + pytest.param({}, id="left-out"), + pytest.param({"x": None, "y": None}, id="null"), + ], + ) + def test__given_breakdown_records_without_x_and_y__then_json_has_no_x_or_y( + self, mock_modal, client: TestClient, map_position: dict[str, None] + ): + # Executors built for the current policyengine.py version send no + # x or y on UK area records; the JSON must not invent null keys. + constituency = _without_x_and_y( + LOCAL_AREA_BREAKDOWN_RESULT["constituency_impact"][0] + ) + local_authority = { + "local_authority_code": "E06000063", + "local_authority_name": "Cumberland", + "average_household_income_change": -120.0, + "relative_household_income_change": -0.002, + "population": 120000.0, + "disclosure_tier": "approximate", + "average_household_income_change_lower": -160.0, + "average_household_income_change_upper": -80.0, + "relative_household_income_change_lower": -0.003, + "relative_household_income_change_upper": -0.001, + } + submit_response = client.post( + "/simulate/economy/comparison", + json={"country": "uk", "scope": "macro", "region": "uk", "reform": {}}, + ) + mock_modal["func"].last_call.result = { + **LOCAL_AREA_BREAKDOWN_RESULT, + "constituency_impact": [{**constituency, **map_position}], + "local_authority_impact": [{**local_authority, **map_position}], + } + + response = client.get(f"/jobs/{submit_response.json()['job_id']}") + + assert response.status_code == 200 + result = response.json()["result"] + assert result["constituency_impact"] == [constituency] + assert result["local_authority_impact"] == [local_authority] + + def test__given_national_records_without_x_and_y__then_json_has_no_x_or_y( + self, mock_modal, client: TestClient + ): + constituency = _without_x_and_y( + LOCAL_AREA_BREAKDOWN_RESULT["constituency_impact"][0] + ) + submit_response = client.post( + "/simulate/economy/comparison", + json={"country": "uk", "scope": "macro", "region": "uk", "reform": {}}, + ) + call = mock_modal["func"].last_call + call.result = {**call.result, "constituency_impact": [constituency]} + + response = client.get(f"/jobs/{submit_response.json()['job_id']}") + + assert response.status_code == 200 + assert response.json()["result"]["constituency_impact"] == [constituency] + def test__given_running_job__then_polling_returns_202( self, mock_modal, client: TestClient ): diff --git a/projects/policyengine-simulation-gateway/uv.lock b/projects/policyengine-simulation-gateway/uv.lock index a17de18fd..be8dfc5bd 100644 --- a/projects/policyengine-simulation-gateway/uv.lock +++ b/projects/policyengine-simulation-gateway/uv.lock @@ -1304,7 +1304,7 @@ requires-dist = [ { name = "modal", marker = "extra == 'test'", specifier = ">=1.4,<2" }, { name = "packaging", specifier = ">=24" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, From 19c8979a7461351a74db97250d60875094f9f3ba Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:02:37 +0400 Subject: [PATCH 2/5] Stop sending x and y on UK area records The executor no longer puts x or y on constituency or local-authority records. It reads only each constituency's code and name from policyengine.py's lookup, whose x and y columns it now ignores, and no longer attaches the packaged local-authority positions. The check that refused a published area without a map position ("missing lookup coordinates") is removed; the code and lookup-name checks stay. With the contract leaving absent x and y out of every dump, the result an app deployed from now on returns has neither key. Apps deployed for older policyengine.py versions are not rebuilt and keep sending both, which the contract still accepts. The beta integration check of a local-area breakdown now requires its records to carry no x or y, since it runs against the app the same deployment builds. Written red-green: the executor tests failed first on the "missing lookup coordinates" error and on x in the constituency and local-authority records, and the breakdown check's new cases failed with "DID NOT RAISE". Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 1 + .../tests/simulation/test_calculate.py | 44 +++++++++---- .../simulation_output_geographic.py | 63 ++++++------------- .../tests/test_simulation_output_builder.py | 4 -- .../test_simulation_output_geographic.py | 34 +++++++--- 5 files changed, 78 insertions(+), 68 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e7a929695..43b79ce3c 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -16,6 +16,7 @@ - Made the budget refuse a household count outside its country's plausible range (above 200 million in the US or 60 million in the UK, negative or not finite), raising ImplausibleEntityCountError instead of publishing a count like #767's changed: - Made x and y, each UK constituency and local-authority record's integer map position, optional in the contract; a record now validates without them and its JSON leaves them out instead of writing null, while records from apps deployed for older policyengine.py versions, which still send both, validate and keep them; the gateway's OpenAPI no longer lists them as required, and the contract now requires pydantic 2.12 or later, the first release with field-level exclude_if + - Stopped sending x and y on UK constituency and local-authority records; the executor no longer reads the x and y columns of policyengine.py's constituency lookup or attaches its packaged local-authority positions, and no longer refuses a published area without a map position, so results from apps deployed from now on carry neither key, while apps deployed for older policyengine.py versions keep sending both; the beta integration check of a local-area breakdown now requires records without them - Removed the UK constituency and local-authority weight-matrix region fallback (_build_uk_weight_replacement_region, _require_certified_uk_weight_matrix and its Stage 12 call); each deployment rebuilds only the app for the current policyengine.py version, now 6.10.0, which routes those regions to the local-area dataset, and apps deployed for older versions are not rebuilt and keep their own code; on a bundle that does not route them, an area region is now an unsupported region - Made UK-wide and nation-level runs on a policyengine.py bundle that routes UK area regions to a local-area release dataset (6.10.0) load only the national dataset and return null for constituency_impact and local_authority_impact, whose results come from a breakdown=local_areas run instead; on a bundle that does not route them, those runs still compute both from the national dataset, and Stage 12 leaves the same breakdowns out of the same reports - Enabled every ruff rule (select ALL), ignoring only COM812 and ISC001, which conflict with ruff format, and the D203 and D213 halves of the mutually exclusive docstring pairs; complex functions, long parameter lists and magic numbers were split into smaller helpers and grouped parameters, each behind new characterization tests, across the contract and persistence validation, entry settings, app factory and Stage 12 adapter, gateway route resolution, Stage 12 coordination, workers, parity and qualification, the precompute and national reduce, the routing publisher and its command line, the release sentinel, the package update helper and the shared rules tool; Modal function, FastAPI route, Pydantic validator and command-line signatures and every OpenAPI document are unchanged; the contract library's batch-state builder and SPM selection resolver take their route values as one grouped argument, the simulation observability initializer drops a service_version parameter no caller passed, and no PLR0913 suppression is needed diff --git a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py index b4559f1fe..cd02f1aff 100644 --- a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py +++ b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py @@ -263,7 +263,8 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: """Check a local-area breakdown came from the local-area dataset. Both area levels must publish at least one area, and every published area - must carry a disclosure tier. + must carry a disclosure tier and no x or y, which only apps deployed for + older policyengine.py versions still send. """ breakdown = _as_mapping(breakdown_result, label="local-area breakdown") assert breakdown.get("kind") == "local_areas", ( @@ -283,6 +284,10 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: assert record.get("disclosure_tier") in _PUBLISHED_DISCLOSURE_TIERS, ( f"Expected a published disclosure_tier in {key}, got {record}" ) + map_position = {"x", "y"} & set(record) + assert not map_position, ( + f"Expected {key} records without x or y, got {record}" + ) class _GeneratedClientObject: @@ -326,8 +331,6 @@ def test_result_assertions_accept_generated_client_objects() -> None: _BREAKDOWN_CONSTITUENCY = { "constituency_code": "E14001063", "constituency_name": "Aldershot", - "x": 56, - "y": -40, "average_household_income_change": -150.0, "relative_household_income_change": -0.0026, "population": 40530.0, @@ -340,8 +343,6 @@ def test_result_assertions_accept_generated_client_objects() -> None: _BREAKDOWN_LOCAL_AUTHORITY = { "local_authority_code": "E06000063", "local_authority_name": "Cumberland", - "x": 4, - "y": 18, "average_household_income_change": -120.0, "relative_household_income_change": -0.002, "population": 120000.0, @@ -362,10 +363,6 @@ def test_result_assertions_accept_generated_client_objects() -> None: } -def _without_x_and_y(record: Mapping) -> dict: - return {key: value for key, value in record.items() if key not in {"x", "y"}} - - def test_breakdown_assertions_accept_generated_client_objects() -> None: status = JobStatusResponse.from_dict( {"status": "complete", "result": _BREAKDOWN_RESULT} @@ -375,12 +372,21 @@ def test_breakdown_assertions_accept_generated_client_objects() -> None: assert_uk_local_area_breakdown(status.result) -def test_generated_client_reads_area_records_without_x_and_y() -> None: - # Executors built for the current policyengine.py version send no x or y. +@pytest.mark.parametrize( + "map_position", + [ + pytest.param({}, id="current-app"), + pytest.param({"x": 56, "y": -40}, id="older-app"), + ], +) +def test_generated_client_reads_area_records_with_and_without_x_and_y( + map_position: dict[str, int], +) -> None: + # Apps deployed for older policyengine.py versions still send x and y. breakdown = { **_BREAKDOWN_RESULT, - "constituency_impact": [_without_x_and_y(_BREAKDOWN_CONSTITUENCY)], - "local_authority_impact": [_without_x_and_y(_BREAKDOWN_LOCAL_AUTHORITY)], + "constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, **map_position}], + "local_authority_impact": [{**_BREAKDOWN_LOCAL_AUTHORITY, **map_position}], } status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown}) @@ -403,6 +409,18 @@ def test_generated_client_reads_area_records_without_x_and_y() -> None: }, "disclosure_tier", ), + ( + {"constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}]}, + "x or y", + ), + ( + { + "local_authority_impact": [ + {**_BREAKDOWN_LOCAL_AUTHORITY, "x": 4, "y": 18} + ] + }, + "x or y", + ), ], ) def test_breakdown_assertions_reject_an_incomplete_breakdown(change, message) -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 4ce4a49b8..241a8c554 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import math from collections.abc import Mapping from dataclasses import dataclass, field from functools import lru_cache @@ -82,7 +81,7 @@ def _complete_uk_geography_output( code_field: str, name_field: str, ) -> GeographicImpactOutput: - """Require every UK geography record to contain lookup-owned metadata. + """Require every UK geography record to carry its code and lookup name. An empty list is valid: disclosure control may withhold every area. """ @@ -100,9 +99,6 @@ def _complete_uk_geography_output( if not isinstance(name, str) or not name.strip() or name == code: msg = f"UK geography {code!r} is missing a lookup name" raise ValueError(msg) - if record_values.get("x") is None or record_values.get("y") is None: - msg = f"UK geography {code!r} is missing lookup coordinates" - raise ValueError(msg) return output @@ -646,10 +642,12 @@ def compute_uk_geography_disclosure_output( ) -def _uk_constituency_lookup( - lookup_csv_path: str, -) -> dict[str, tuple[str, int | None, int | None]]: - """Return each constituency's name and map coordinates, in lookup order.""" +def _uk_constituency_lookup(lookup_csv_path: str) -> dict[str, str]: + """Return each constituency's name, in lookup order. + + The lookup's other columns, such as its x and y map positions, are not + read: sim-api results no longer carry an area's map position. + """ lookup = pd.read_csv(lookup_csv_path) if "code" not in lookup.columns: msg = ( @@ -657,28 +655,13 @@ def _uk_constituency_lookup( f"{lookup_csv_path}" ) raise ValueError(msg) - entries: dict[str, tuple[str, int | None, int | None]] = {} + names: dict[str, str] = {} for row in lookup.to_dict("records"): code = _normalised_area_code(row.get("code")) if not code: continue - name = _normalised_area_code(row.get("name")) or code - entries[code] = ( - name, - _lookup_coordinate(row.get("x")), - _lookup_coordinate(row.get("y")), - ) - return entries - - -def _lookup_coordinate(value: object) -> int | None: - if isinstance(value, bool) or not isinstance(value, int | float | str): - return None - try: - number = float(value) - except ValueError: - return None - return int(number) if math.isfinite(number) else None + names[code] = _normalised_area_code(row.get("name")) or code + return names def _require_uk_disclosure_output( @@ -699,7 +682,7 @@ def build_uk_constituency_impact( Areas that disclosure control withholds are left out. Downloads the constituency lookup if needed and raises when a published area lacks a - lookup name or coordinates. + lookup name. """ if country != "uk": return None @@ -712,18 +695,14 @@ def build_uk_constituency_impact( ordered = [code for code in lookup if code in changes] + sorted( set(changes) - set(lookup) ) - records: list[dict[str, object]] = [] - for code in ordered: - name, x, y = lookup.get(code, (code, None, None)) - records.append( - { - "constituency_code": code, - "constituency_name": name, - "x": x, - "y": y, - **changes[code].record_fields(), - } - ) + records: list[dict[str, object]] = [ + { + "constituency_code": code, + "constituency_name": lookup.get(code, code), + **changes[code].record_fields(), + } + for code in ordered + ] return _complete_uk_geography_output( records, code_field="constituency_code", @@ -737,7 +716,7 @@ def build_uk_local_authority_impact( *, uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> GeographicImpactOutput | None: - """Return the published UK local authorities, named and placed for the boundaries. + """Return the published UK local authorities, named for the boundary version. None for other countries. For the UK, ``uk_local_authority_metadata`` is required, and a code outside its boundary version raises ``ValueError``. @@ -773,8 +752,6 @@ def build_uk_local_authority_impact( { "local_authority_code": code, "local_authority_name": metadata.name, - "x": metadata.x, - "y": metadata.y, **changes[code].record_fields(), } ) diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py index 44b99a136..dffa86b89 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py @@ -289,8 +289,6 @@ def _constituency_impact_record() -> ConstituencyImpactRecord: return ConstituencyImpactRecord( constituency_code="E14000530", constituency_name="Birmingham, Ladywood", - x=34, - y=-42, average_household_income_change=10.0, relative_household_income_change=0.01, population=1000.0, @@ -301,8 +299,6 @@ def _local_authority_impact_record() -> LocalAuthorityImpactRecord: return LocalAuthorityImpactRecord( local_authority_code="E06000001", local_authority_name="Hartlepool", - x=63, - y=-19, average_household_income_change=12.0, relative_household_income_change=0.02, population=900.0, diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index 84636b533..05b27f12b 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -22,8 +22,6 @@ def _complete_constituency_record() -> dict[str, object]: return { "constituency_code": "E14001063", "constituency_name": "Aldershot", - "x": 56, - "y": -40, "average_household_income_change": -150.95, "relative_household_income_change": -0.0026, "population": 40_530.65, @@ -85,8 +83,8 @@ def test_complete_uk_geography_output_accepts_lookup_metadata() -> None: record = result.root[0].model_dump(mode="python") assert record["constituency_name"] == "Aldershot" - assert record["x"] == 56 - assert record["y"] == -40 + assert "x" not in record + assert "y" not in record def test_complete_uk_geography_output_accepts_an_empty_list() -> None: @@ -114,8 +112,6 @@ def test_complete_uk_geography_output_rejects_a_missing_list() -> None: [ ("constituency_name", "E14001063", "missing a lookup name"), ("constituency_name", "", "missing a lookup name"), - ("x", None, "missing lookup coordinates"), - ("y", None, "missing lookup coordinates"), ], ) def test_complete_uk_geography_output_rejects_fallback_metadata( @@ -286,6 +282,10 @@ def test_uk_constituency_output_lists_only_published_seats( for record in records: change = published[record["constituency_code"]] assert record["constituency_name"].startswith("Seat ") + # policyengine.py's lookup still has x and y columns; sim-api no + # longer sends an area's map position. + assert "x" not in record + assert "y" not in record assert record["average_household_income_change"] == change.average assert record["relative_household_income_change"] == change.relative assert record["disclosure_tier"] == change.tier @@ -322,14 +322,32 @@ def test_uk_local_authority_output_uses_detected_boundary_metadata( "Cumberland", "Westmorland and Furness", } - assert isinstance(record["x"], int) - assert isinstance(record["y"], int) + assert "x" not in record + assert "y" not in record assert record["average_household_income_change"] < 0 assert record["population"] == ( uk_disclosure.level("local_authority")[code].population ) +def test_uk_area_outputs_serialize_without_x_or_y( + uk_disclosure, constituency_lookup +) -> None: + constituencies = simulation_output_geographic.build_uk_constituency_impact( + "uk", uk_disclosure + ) + authorities = simulation_output_geographic.build_uk_local_authority_impact( + "uk", uk_disclosure, uk_local_authority_metadata=LAD23 + ) + + assert constituencies is not None + assert authorities is not None + for output in (constituencies, authorities): + records = output.model_dump(mode="json") + assert records + assert all("x" not in record and "y" not in record for record in records) + + def test_uk_local_authority_output_rejects_code_outside_boundary_version( uk_disclosure, ) -> None: From 981016b339e80663706f478264383cd8e04572e6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:06:54 +0400 Subject: [PATCH 3/5] Ship local-authority code lists instead of coordinate files The executor no longer ships coordinates_lad22.csv or coordinates_lad23.csv. Nothing reads their x and y any more, but they were also the only list of each boundary version's authority codes, so codes_lad22.csv and codes_lad23.csv replace them with a single code column holding exactly the same codes in the same order. UKLocalAuthorityResources now carries those codes per boundary version (codes) and returns one version's names with names_for, which replaces metadata_for. The load-time checks are unchanged apart from the dropped coordinate comparison: names must exactly cover both lists, and each version's distinguishing codes must match its list. LAD22 and LAD23 detection reads the names and the distinguishing code sets, so it is unchanged. The Stage 12 static runtime file smoke reports lad22_codes and lad23_codes in place of lad22_coordinates and lad23_coordinates, with the same expected counts (374 and 361). A test pins each version's codes to a SHA-256 of the codes the coordinate files listed, and new tests check the loader refuses a repeated code, a list with other distinguishing codes, and a list with x and y columns. Written red-green: the tests failed first because the resources had no codes or names_for, the package still held the coordinate files and no code lists, and the smoke reported the coordinate keys. Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 1 + .../stage-12-modal-worker-foundation.md | 38 +- .../modal/v2_static_runtime_files_smoke.py | 12 +- .../simulation_output_geographic.py | 10 +- .../stage12_worker_validation.py | 4 +- .../uk_local_authorities/codes_lad22.csv | 375 ++++++++++++++++++ .../uk_local_authorities/codes_lad23.csv | 362 +++++++++++++++++ .../coordinates_lad22.csv | 375 ------------------ .../coordinates_lad23.csv | 362 ----------------- .../uk_local_authority_metadata.py | 87 ++-- .../tests/test_stage12_modal_app.py | 4 +- .../tests/test_uk_local_authority_metadata.py | 148 ++++++- 12 files changed, 925 insertions(+), 853 deletions(-) create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv delete mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv delete mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 43b79ce3c..d9042e4e0 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -17,6 +17,7 @@ changed: - Made x and y, each UK constituency and local-authority record's integer map position, optional in the contract; a record now validates without them and its JSON leaves them out instead of writing null, while records from apps deployed for older policyengine.py versions, which still send both, validate and keep them; the gateway's OpenAPI no longer lists them as required, and the contract now requires pydantic 2.12 or later, the first release with field-level exclude_if - Stopped sending x and y on UK constituency and local-authority records; the executor no longer reads the x and y columns of policyengine.py's constituency lookup or attaches its packaged local-authority positions, and no longer refuses a published area without a map position, so results from apps deployed from now on carry neither key, while apps deployed for older policyengine.py versions keep sending both; the beta integration check of a local-area breakdown now requires records without them + - Stopped shipping coordinates_lad22.csv and coordinates_lad23.csv in the executor; each boundary version's authority codes now come from codes_lad22.csv and codes_lad23.csv, which list exactly the codes the coordinate files did, so the LAD22 and LAD23 detection and its checks (names cover both lists, and each version's distinguishing codes match its list) are unchanged, and the Stage 12 static runtime file smoke reports lad22_codes and lad23_codes instead of lad22_coordinates and lad23_coordinates - Removed the UK constituency and local-authority weight-matrix region fallback (_build_uk_weight_replacement_region, _require_certified_uk_weight_matrix and its Stage 12 call); each deployment rebuilds only the app for the current policyengine.py version, now 6.10.0, which routes those regions to the local-area dataset, and apps deployed for older versions are not rebuilt and keep their own code; on a bundle that does not route them, an area region is now an unsupported region - Made UK-wide and nation-level runs on a policyengine.py bundle that routes UK area regions to a local-area release dataset (6.10.0) load only the national dataset and return null for constituency_impact and local_authority_impact, whose results come from a breakdown=local_areas run instead; on a bundle that does not route them, those runs still compute both from the national dataset, and Stage 12 leaves the same breakdowns out of the same reports - Enabled every ruff rule (select ALL), ignoring only COM812 and ISC001, which conflict with ruff format, and the D203 and D213 halves of the mutually exclusive docstring pairs; complex functions, long parameter lists and magic numbers were split into smaller helpers and grouped parameters, each behind new characterization tests, across the contract and persistence validation, entry settings, app factory and Stage 12 adapter, gateway route resolution, Stage 12 coordination, workers, parity and qualification, the precompute and national reduce, the routing publisher and its command line, the release sentinel, the package update helper and the shared rules tool; Modal function, FastAPI route, Pydantic validator and command-line signatures and every OpenAPI document are unchanged; the contract library's batch-state builder and SPM selection resolver take their route values as one grouped argument, the simulation observability initializer drops a service_version parameter no caller passed, and no PLR0913 suppression is needed diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 4ecb5fc0e..347e1f334 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -255,21 +255,29 @@ configuration, an unknown code, or a dataset without a distinguishing code fails the Stage 12 run. Dataset names and release labels do not select a boundary version, and there is no fallback to the newest known configuration. -The executor temporarily packages three display-only resources under its -expandable `static_runtime_files/uk_local_authorities/` directory. The v2 Modal -image copies that directory explicitly because Modal's Python-source layer does -not include non-Python files by default: - -- one common `code,name` file containing the union of supported authority - codes; -- one `code,x,y` file for the 374-authority LAD22 configuration; and -- one `code,x,y` file for the 361-authority LAD23 configuration. +The executor temporarily packages three resources under its expandable +`static_runtime_files/uk_local_authorities/` directory. The v2 Modal image +copies that directory explicitly because Modal's Python-source layer does not +include non-Python files by default: + +- one common `code,name` file, `names.csv`, containing the union of supported + authority codes; +- one `code` file, `codes_lad22.csv`, listing the 374-authority LAD22 + configuration; and +- one `code` file, `codes_lad23.csv`, listing the 361-authority LAD23 + configuration. + +The two code lists replaced `coordinates_lad22.csv` and +`coordinates_lad23.csv`, which also gave each authority an integer map +position (`x` and `y`). Results no longer carry a map position, so the +executor no longer ships one; the code lists keep exactly the codes those +files listed. The Enhanced FRS geography is LAD22: its 2021 output areas are crosswalked to 2022 local-authority boundaries. The output-area vintage does not make the -local-authority boundary version LAD21. Names and coordinates are attached only -after numeric local-authority impacts have been calculated; these resources -never assign a household to an authority. +local-authority boundary version LAD21. Names are attached only after numeric +local-authority impacts have been calculated; these resources never assign a +household to an authority. The detected boundary version is stored as typed metadata in both child Parquet artifacts. The coordinator requires baseline and reform to agree before it @@ -277,9 +285,9 @@ enriches the aggregate output. Existing non-Stage-12 calculations continue to use the existing GCS lookup and are not changed by this path. This resource ownership is temporary. A future boundary version must add its -codes to the common names file, add a distinct coordinate file, and register a -typed detector definition. Longer term, display-coordinate rendering should -move to the front end, and Stage 14 or later work should remove these temporary +codes to the common names file, add a distinct code file, and register a typed +detector definition. Map positions are now the front end's to draw from its +own layout files, and Stage 14 or later work should remove these temporary executor resources when the authoritative v2 report architecture supersedes Stage 12. diff --git a/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py b/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py index 791f99e15..599e466ae 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py @@ -44,12 +44,8 @@ def smoke_stage12_static_runtime_files() -> dict[str, int | str]: resources = load_uk_local_authority_resources() return { "names": len(resources.names), - "lad22_coordinates": len( - resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - ), - "lad23_coordinates": len( - resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] - ), + "lad22_codes": len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD22]), + "lad23_codes": len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD23]), "installed_dataset_boundary_version": ( validate_installed_uk_local_authority_dataset() ), @@ -62,8 +58,8 @@ def main() -> None: observed = smoke_stage12_static_runtime_files.remote() expected_counts = { "names": 378, - "lad22_coordinates": 374, - "lad23_coordinates": 361, + "lad22_codes": 374, + "lad23_codes": 361, } if {key: observed.get(key) for key in expected_counts} != expected_counts: msg = ( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 241a8c554..e79339525 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -736,13 +736,11 @@ def build_uk_local_authority_impact( ) boundary_version = uk_local_authority_metadata.boundary_version - display_metadata = load_uk_local_authority_resources().metadata_for( - boundary_version - ) + names = load_uk_local_authority_resources().names_for(boundary_version) records: list[dict[str, object]] = [] for code in sorted(changes): - metadata = display_metadata.get(code) - if metadata is None: + name = names.get(code) + if name is None: msg = ( f"UK local-authority code {code!r} is not part of the detected " f"{boundary_version.value.upper()} boundary version" @@ -751,7 +749,7 @@ def build_uk_local_authority_impact( records.append( { "local_authority_code": code, - "local_authority_name": metadata.name, + "local_authority_name": name, **changes[code].record_fields(), } ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py index 102593c99..a20614d64 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py @@ -122,8 +122,8 @@ def _check_uk_local_authority_resources() -> None: ) resources = load_uk_local_authority_resources() - for boundary_version in resources.coordinates: - resources.metadata_for(boundary_version) + for boundary_version in resources.codes: + resources.names_for(boundary_version) def _check_uk_local_authority_dataset(dataset_path: str) -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv new file mode 100644 index 000000000..111f7f79d --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv @@ -0,0 +1,375 @@ +code +E06000001 +E06000002 +E06000003 +E06000004 +E06000005 +E06000006 +E06000007 +E06000008 +E06000009 +E06000010 +E06000011 +E06000012 +E06000013 +E06000014 +E06000015 +E06000016 +E06000017 +E06000018 +E06000019 +E06000020 +E06000021 +E06000022 +E06000023 +E06000024 +E06000025 +E06000026 +E06000027 +E06000030 +E06000031 +E06000032 +E06000033 +E06000034 +E06000035 +E06000036 +E06000037 +E06000038 +E06000039 +E06000040 +E06000041 +E06000042 +E06000043 +E06000044 +E06000045 +E06000046 +E06000047 +E06000049 +E06000050 +E06000051 +E06000052 +E06000053 +E06000054 +E06000055 +E06000056 +E06000057 +E06000058 +E06000059 +E06000060 +E06000061 +E06000062 +E07000008 +E07000009 +E07000010 +E07000011 +E07000012 +E07000026 +E07000027 +E07000028 +E07000029 +E07000030 +E07000031 +E07000032 +E07000033 +E07000034 +E07000035 +E07000036 +E07000037 +E07000038 +E07000039 +E07000040 +E07000041 +E07000042 +E07000043 +E07000044 +E07000045 +E07000046 +E07000047 +E07000061 +E07000062 +E07000063 +E07000064 +E07000065 +E07000066 +E07000067 +E07000068 +E07000069 +E07000070 +E07000071 +E07000072 +E07000073 +E07000074 +E07000075 +E07000076 +E07000077 +E07000078 +E07000079 +E07000080 +E07000081 +E07000082 +E07000083 +E07000084 +E07000085 +E07000086 +E07000087 +E07000088 +E07000089 +E07000090 +E07000091 +E07000092 +E07000093 +E07000094 +E07000095 +E07000096 +E07000098 +E07000099 +E07000102 +E07000103 +E07000105 +E07000106 +E07000107 +E07000108 +E07000109 +E07000110 +E07000111 +E07000112 +E07000113 +E07000114 +E07000115 +E07000116 +E07000117 +E07000118 +E07000119 +E07000120 +E07000121 +E07000122 +E07000123 +E07000124 +E07000125 +E07000126 +E07000127 +E07000128 +E07000129 +E07000130 +E07000131 +E07000132 +E07000133 +E07000134 +E07000135 +E07000136 +E07000137 +E07000138 +E07000139 +E07000140 +E07000141 +E07000142 +E07000143 +E07000144 +E07000145 +E07000146 +E07000147 +E07000148 +E07000149 +E07000163 +E07000164 +E07000165 +E07000166 +E07000167 +E07000168 +E07000169 +E07000170 +E07000171 +E07000172 +E07000173 +E07000174 +E07000175 +E07000176 +E07000177 +E07000178 +E07000179 +E07000180 +E07000181 +E07000187 +E07000188 +E07000189 +E07000192 +E07000193 +E07000194 +E07000195 +E07000196 +E07000197 +E07000198 +E07000199 +E07000200 +E07000202 +E07000203 +E07000207 +E07000208 +E07000209 +E07000210 +E07000211 +E07000212 +E07000213 +E07000214 +E07000215 +E07000216 +E07000217 +E07000218 +E07000219 +E07000220 +E07000221 +E07000222 +E07000223 +E07000224 +E07000225 +E07000226 +E07000227 +E07000228 +E07000229 +E07000234 +E07000235 +E07000236 +E07000237 +E07000238 +E07000239 +E07000240 +E07000241 +E07000242 +E07000243 +E07000244 +E07000245 +E07000246 +E08000001 +E08000002 +E08000003 +E08000004 +E08000005 +E08000006 +E08000007 +E08000008 +E08000009 +E08000010 +E08000011 +E08000012 +E08000013 +E08000014 +E08000015 +E08000016 +E08000017 +E08000018 +E08000019 +E08000021 +E08000022 +E08000023 +E08000024 +E08000025 +E08000026 +E08000027 +E08000028 +E08000029 +E08000030 +E08000031 +E08000032 +E08000033 +E08000034 +E08000035 +E08000036 +E08000037 +E09000001 +E09000002 +E09000003 +E09000004 +E09000005 +E09000006 +E09000007 +E09000008 +E09000009 +E09000010 +E09000011 +E09000012 +E09000013 +E09000014 +E09000015 +E09000016 +E09000017 +E09000018 +E09000019 +E09000020 +E09000021 +E09000022 +E09000023 +E09000024 +E09000025 +E09000026 +E09000027 +E09000028 +E09000029 +E09000030 +E09000031 +E09000032 +E09000033 +N09000001 +N09000002 +N09000003 +N09000004 +N09000005 +N09000006 +N09000007 +N09000008 +N09000009 +N09000010 +N09000011 +S12000005 +S12000006 +S12000008 +S12000010 +S12000011 +S12000013 +S12000014 +S12000017 +S12000018 +S12000019 +S12000020 +S12000021 +S12000023 +S12000026 +S12000027 +S12000028 +S12000029 +S12000030 +S12000033 +S12000034 +S12000035 +S12000036 +S12000038 +S12000039 +S12000040 +S12000041 +S12000042 +S12000045 +S12000047 +S12000048 +S12000049 +S12000050 +W06000001 +W06000002 +W06000003 +W06000004 +W06000005 +W06000006 +W06000008 +W06000009 +W06000010 +W06000011 +W06000012 +W06000013 +W06000014 +W06000015 +W06000016 +W06000018 +W06000019 +W06000020 +W06000021 +W06000022 +W06000023 +W06000024 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv new file mode 100644 index 000000000..f184d3a26 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv @@ -0,0 +1,362 @@ +code +E06000001 +E06000002 +E06000003 +E06000004 +E06000005 +E06000006 +E06000007 +E06000008 +E06000009 +E06000010 +E06000011 +E06000012 +E06000013 +E06000014 +E06000015 +E06000016 +E06000017 +E06000018 +E06000019 +E06000020 +E06000021 +E06000022 +E06000023 +E06000024 +E06000025 +E06000026 +E06000027 +E06000030 +E06000031 +E06000032 +E06000033 +E06000034 +E06000035 +E06000036 +E06000037 +E06000038 +E06000039 +E06000040 +E06000041 +E06000042 +E06000043 +E06000044 +E06000045 +E06000046 +E06000047 +E06000049 +E06000050 +E06000051 +E06000052 +E06000053 +E06000054 +E06000055 +E06000056 +E06000057 +E06000058 +E06000059 +E06000060 +E06000061 +E06000062 +E06000063 +E06000064 +E06000065 +E06000066 +E07000008 +E07000009 +E07000010 +E07000011 +E07000012 +E07000032 +E07000033 +E07000034 +E07000035 +E07000036 +E07000037 +E07000038 +E07000039 +E07000040 +E07000041 +E07000042 +E07000043 +E07000044 +E07000045 +E07000046 +E07000047 +E07000061 +E07000062 +E07000063 +E07000064 +E07000065 +E07000066 +E07000067 +E07000068 +E07000069 +E07000070 +E07000071 +E07000072 +E07000073 +E07000074 +E07000075 +E07000076 +E07000077 +E07000078 +E07000079 +E07000080 +E07000081 +E07000082 +E07000083 +E07000084 +E07000085 +E07000086 +E07000087 +E07000088 +E07000089 +E07000090 +E07000091 +E07000092 +E07000093 +E07000094 +E07000095 +E07000096 +E07000098 +E07000099 +E07000102 +E07000103 +E07000105 +E07000106 +E07000107 +E07000108 +E07000109 +E07000110 +E07000111 +E07000112 +E07000113 +E07000114 +E07000115 +E07000116 +E07000117 +E07000118 +E07000119 +E07000120 +E07000121 +E07000122 +E07000123 +E07000124 +E07000125 +E07000126 +E07000127 +E07000128 +E07000129 +E07000130 +E07000131 +E07000132 +E07000133 +E07000134 +E07000135 +E07000136 +E07000137 +E07000138 +E07000139 +E07000140 +E07000141 +E07000142 +E07000143 +E07000144 +E07000145 +E07000146 +E07000147 +E07000148 +E07000149 +E07000170 +E07000171 +E07000172 +E07000173 +E07000174 +E07000175 +E07000176 +E07000177 +E07000178 +E07000179 +E07000180 +E07000181 +E07000192 +E07000193 +E07000194 +E07000195 +E07000196 +E07000197 +E07000198 +E07000199 +E07000200 +E07000202 +E07000203 +E07000207 +E07000208 +E07000209 +E07000210 +E07000211 +E07000212 +E07000213 +E07000214 +E07000215 +E07000216 +E07000217 +E07000218 +E07000219 +E07000220 +E07000221 +E07000222 +E07000223 +E07000224 +E07000225 +E07000226 +E07000227 +E07000228 +E07000229 +E07000234 +E07000235 +E07000236 +E07000237 +E07000238 +E07000239 +E07000240 +E07000241 +E07000242 +E07000243 +E07000244 +E07000245 +E08000001 +E08000002 +E08000003 +E08000004 +E08000005 +E08000006 +E08000007 +E08000008 +E08000009 +E08000010 +E08000011 +E08000012 +E08000013 +E08000014 +E08000015 +E08000016 +E08000017 +E08000018 +E08000019 +E08000021 +E08000022 +E08000023 +E08000024 +E08000025 +E08000026 +E08000027 +E08000028 +E08000029 +E08000030 +E08000031 +E08000032 +E08000033 +E08000034 +E08000035 +E08000036 +E08000037 +E09000001 +E09000002 +E09000003 +E09000004 +E09000005 +E09000006 +E09000007 +E09000008 +E09000009 +E09000010 +E09000011 +E09000012 +E09000013 +E09000014 +E09000015 +E09000016 +E09000017 +E09000018 +E09000019 +E09000020 +E09000021 +E09000022 +E09000023 +E09000024 +E09000025 +E09000026 +E09000027 +E09000028 +E09000029 +E09000030 +E09000031 +E09000032 +E09000033 +N09000001 +N09000002 +N09000003 +N09000004 +N09000005 +N09000006 +N09000007 +N09000008 +N09000009 +N09000010 +N09000011 +S12000005 +S12000006 +S12000008 +S12000010 +S12000011 +S12000013 +S12000014 +S12000017 +S12000018 +S12000019 +S12000020 +S12000021 +S12000023 +S12000026 +S12000027 +S12000028 +S12000029 +S12000030 +S12000033 +S12000034 +S12000035 +S12000036 +S12000038 +S12000039 +S12000040 +S12000041 +S12000042 +S12000045 +S12000047 +S12000048 +S12000049 +S12000050 +W06000001 +W06000002 +W06000003 +W06000004 +W06000005 +W06000006 +W06000008 +W06000009 +W06000010 +W06000011 +W06000012 +W06000013 +W06000014 +W06000015 +W06000016 +W06000018 +W06000019 +W06000020 +W06000021 +W06000022 +W06000023 +W06000024 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv deleted file mode 100644 index cc333e371..000000000 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv +++ /dev/null @@ -1,375 +0,0 @@ -code,x,y -E06000001,8,19 -E06000002,9,18 -E06000003,9,19 -E06000004,8,18 -E06000005,7,18 -E06000006,1,11 -E06000007,2,11 -E06000008,4,15 -E06000009,2,15 -E06000010,10,15 -E06000011,11,16 -E06000012,11,14 -E06000013,10,14 -E06000014,9,17 -E06000015,6,11 -E06000016,8,8 -E06000017,10,9 -E06000018,8,10 -E06000019,0,8 -E06000020,2,9 -E06000021,3,10 -E06000022,1,3 -E06000023,0,3 -E06000024,0,2 -E06000025,1,4 -E06000026,-4,-2 -E06000027,-3,-2 -E06000030,2,4 -E06000031,11,9 -E06000032,10,7 -E06000033,16,6 -E06000034,15,4 -E06000035,15,1 -E06000036,4,2 -E06000037,2,2 -E06000038,2,3 -E06000039,6,4 -E06000040,4,3 -E06000041,3,3 -E06000042,6,5 -E06000043,9,-2 -E06000044,4,-1 -E06000045,2,0 -E06000046,1,-2 -E06000047,6,18 -E06000049,4,11 -E06000050,3,11 -E06000051,1,9 -E06000052,-5,-2 -E06000053,-7,-3 -E06000054,1,2 -E06000055,9,7 -E06000056,9,6 -E06000057,5,20 -E06000058,0,0 -E06000059,-1,0 -E06000060,5,5 -E06000061,9,9 -E06000062,7,6 -E07000008,12,8 -E07000009,12,9 -E07000010,13,10 -E07000011,10,8 -E07000012,11,8 -E07000026,4,18 -E07000027,2,17 -E07000028,4,19 -E07000029,3,18 -E07000030,5,18 -E07000031,4,17 -E07000032,7,11 -E07000033,10,12 -E07000034,9,12 -E07000035,7,12 -E07000036,7,9 -E07000037,7,13 -E07000038,8,12 -E07000039,6,10 -E07000040,-2,-1 -E07000041,-3,-1 -E07000042,-2,0 -E07000043,-3,1 -E07000044,-4,-3 -E07000045,-2,-2 -E07000046,-4,-1 -E07000047,-3,0 -E07000061,10,-2 -E07000062,13,-2 -E07000063,10,-1 -E07000064,12,-2 -E07000065,11,-2 -E07000066,14,5 -E07000067,14,7 -E07000068,13,5 -E07000069,15,5 -E07000070,14,6 -E07000071,15,8 -E07000072,12,5 -E07000073,13,6 -E07000074,15,7 -E07000075,15,6 -E07000076,16,8 -E07000077,13,7 -E07000078,1,5 -E07000079,2,5 -E07000080,-1,6 -E07000081,0,6 -E07000082,0,5 -E07000083,1,6 -E07000084,2,1 -E07000085,4,0 -E07000086,3,0 -E07000087,2,-1 -E07000088,3,-1 -E07000089,3,2 -E07000090,5,0 -E07000091,1,0 -E07000092,4,1 -E07000093,1,1 -E07000094,3,1 -E07000095,12,6 -E07000096,8,6 -E07000098,9,5 -E07000099,11,7 -E07000102,7,5 -E07000103,8,5 -E07000105,12,-1 -E07000106,15,0 -E07000107,13,1 -E07000108,14,-1 -E07000109,14,1 -E07000110,14,0 -E07000111,12,0 -E07000112,13,-1 -E07000113,16,0 -E07000114,15,-1 -E07000115,13,0 -E07000116,11,-1 -E07000117,6,15 -E07000118,3,14 -E07000119,4,16 -E07000120,5,15 -E07000121,3,17 -E07000122,6,16 -E07000123,5,16 -E07000124,5,17 -E07000125,6,14 -E07000126,3,15 -E07000127,2,13 -E07000128,3,16 -E07000129,7,7 -E07000130,8,9 -E07000131,8,7 -E07000132,7,8 -E07000133,11,10 -E07000134,6,9 -E07000135,9,8 -E07000136,12,12 -E07000137,12,13 -E07000138,11,12 -E07000139,11,11 -E07000140,12,11 -E07000141,12,10 -E07000142,11,13 -E07000143,14,10 -E07000144,15,12 -E07000145,15,11 -E07000146,13,11 -E07000147,14,12 -E07000148,14,11 -E07000149,15,10 -E07000163,6,17 -E07000164,10,18 -E07000165,8,17 -E07000166,7,17 -E07000167,10,16 -E07000168,10,17 -E07000169,9,16 -E07000170,8,11 -E07000171,10,13 -E07000172,7,10 -E07000173,9,10 -E07000174,9,11 -E07000175,10,11 -E07000176,10,10 -E07000177,4,5 -E07000178,4,4 -E07000179,5,4 -E07000180,3,4 -E07000181,3,5 -E07000187,0,1 -E07000188,-1,2 -E07000189,-1,1 -E07000192,3,9 -E07000193,5,11 -E07000194,4,9 -E07000195,2,10 -E07000196,2,8 -E07000197,4,10 -E07000198,5,10 -E07000199,5,9 -E07000200,14,8 -E07000202,15,9 -E07000203,14,9 -E07000207,7,2 -E07000208,8,0 -E07000209,5,1 -E07000210,6,1 -E07000211,7,0 -E07000212,5,3 -E07000213,6,3 -E07000214,5,2 -E07000215,9,-1 -E07000216,6,0 -E07000217,6,2 -E07000218,6,8 -E07000219,6,7 -E07000220,6,6 -E07000221,3,6 -E07000222,4,6 -E07000223,8,-2 -E07000224,6,-2 -E07000225,5,-1 -E07000226,8,-1 -E07000227,6,-1 -E07000228,7,-1 -E07000229,7,-2 -E07000234,2,7 -E07000235,-1,7 -E07000236,4,7 -E07000237,0,7 -E07000238,2,6 -E07000239,1,8 -E07000240,10,6 -E07000241,11,6 -E07000242,13,8 -E07000243,12,7 -E07000244,16,10 -E07000245,13,9 -E07000246,-2,1 -E08000001,4,14 -E08000002,5,14 -E08000003,5,12 -E08000004,5,13 -E08000005,7,14 -E08000006,4,13 -E08000007,6,12 -E08000008,6,13 -E08000009,4,12 -E08000010,3,13 -E08000011,2,12 -E08000012,1,13 -E08000013,3,12 -E08000014,2,14 -E08000015,1,12 -E08000016,8,14 -E08000017,9,14 -E08000018,9,13 -E08000019,8,13 -E08000021,5,19 -E08000022,6,20 -E08000023,7,20 -E08000024,7,19 -E08000025,5,8 -E08000026,5,6 -E08000027,1,7 -E08000028,3,7 -E08000029,5,7 -E08000030,4,8 -E08000031,3,8 -E08000032,7,16 -E08000033,7,15 -E08000034,8,15 -E08000035,8,16 -E08000036,9,15 -E08000037,6,19 -E09000001,11,2 -E09000002,13,3 -E09000003,10,5 -E09000004,12,1 -E09000005,10,4 -E09000006,11,0 -E09000007,11,4 -E09000008,10,0 -E09000009,9,4 -E09000010,11,5 -E09000011,11,1 -E09000012,12,3 -E09000013,8,3 -E09000014,12,4 -E09000015,8,4 -E09000016,14,3 -E09000017,7,4 -E09000018,7,3 -E09000019,11,3 -E09000020,9,3 -E09000021,7,1 -E09000022,10,2 -E09000023,10,1 -E09000024,8,1 -E09000025,13,2 -E09000026,14,4 -E09000027,8,2 -E09000028,9,1 -E09000029,9,0 -E09000030,12,2 -E09000031,13,4 -E09000032,9,2 -E09000033,10,3 -N09000001,-4,16 -N09000002,-5,16 -N09000003,-4,17 -N09000004,-5,18 -N09000005,-6,17 -N09000006,-6,16 -N09000007,-5,15 -N09000008,-4,18 -N09000009,-5,17 -N09000010,-4,15 -N09000011,-3,16 -S12000005,2,24 -S12000006,4,20 -S12000008,3,20 -S12000010,5,22 -S12000011,2,20 -S12000013,-1,27 -S12000014,2,23 -S12000017,1,26 -S12000018,0,21 -S12000019,3,21 -S12000020,2,26 -S12000021,1,20 -S12000023,4,28 -S12000026,4,21 -S12000027,5,30 -S12000028,1,19 -S12000029,2,21 -S12000030,1,24 -S12000033,4,26 -S12000034,3,26 -S12000035,0,24 -S12000036,4,22 -S12000038,1,22 -S12000039,0,23 -S12000040,3,22 -S12000041,2,25 -S12000042,3,25 -S12000045,1,23 -S12000047,3,24 -S12000048,1,25 -S12000049,1,21 -S12000050,2,22 -W06000001,-2,12 -W06000002,-2,10 -W06000003,-1,10 -W06000004,0,10 -W06000005,0,11 -W06000006,1,10 -W06000008,-2,9 -W06000009,-5,6 -W06000010,-4,6 -W06000011,-4,5 -W06000012,-3,5 -W06000013,-3,6 -W06000014,-2,4 -W06000015,-2,5 -W06000016,-3,7 -W06000018,-2,6 -W06000019,0,9 -W06000020,-2,7 -W06000021,-1,8 -W06000022,-1,5 -W06000023,-1,9 -W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv deleted file mode 100644 index 591d36189..000000000 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv +++ /dev/null @@ -1,362 +0,0 @@ -code,x,y -E06000001,8,19 -E06000002,9,18 -E06000003,9,19 -E06000004,8,18 -E06000005,7,18 -E06000006,1,11 -E06000007,2,11 -E06000008,4,15 -E06000009,2,15 -E06000010,10,15 -E06000011,11,16 -E06000012,11,14 -E06000013,10,14 -E06000014,9,17 -E06000015,6,11 -E06000016,8,8 -E06000017,10,9 -E06000018,8,10 -E06000019,0,8 -E06000020,2,9 -E06000021,3,10 -E06000022,1,3 -E06000023,0,3 -E06000024,0,2 -E06000025,1,4 -E06000026,-4,-2 -E06000027,-3,-2 -E06000030,2,4 -E06000031,11,9 -E06000032,10,7 -E06000033,16,6 -E06000034,15,4 -E06000035,15,1 -E06000036,4,2 -E06000037,2,2 -E06000038,2,3 -E06000039,6,4 -E06000040,4,3 -E06000041,3,3 -E06000042,6,5 -E06000043,9,-2 -E06000044,4,-1 -E06000045,2,0 -E06000046,1,-2 -E06000047,6,18 -E06000049,4,11 -E06000050,3,11 -E06000051,1,9 -E06000052,-5,-2 -E06000053,-7,-3 -E06000054,1,2 -E06000055,9,7 -E06000056,9,6 -E06000057,5,20 -E06000058,0,0 -E06000059,-1,0 -E06000060,5,5 -E06000061,9,9 -E06000062,7,6 -E06000063,4,18 -E06000064,4,17 -E06000065,8,17 -E06000066,0,1 -E07000008,12,8 -E07000009,12,9 -E07000010,13,10 -E07000011,10,8 -E07000012,11,8 -E07000032,7,11 -E07000033,10,12 -E07000034,9,12 -E07000035,7,12 -E07000036,7,9 -E07000037,7,13 -E07000038,8,12 -E07000039,6,10 -E07000040,-2,-1 -E07000041,-3,-1 -E07000042,-2,0 -E07000043,-3,1 -E07000044,-4,-3 -E07000045,-2,-2 -E07000046,-4,-1 -E07000047,-3,0 -E07000061,10,-2 -E07000062,13,-2 -E07000063,10,-1 -E07000064,12,-2 -E07000065,11,-2 -E07000066,14,5 -E07000067,14,7 -E07000068,13,5 -E07000069,15,5 -E07000070,14,6 -E07000071,15,8 -E07000072,12,5 -E07000073,13,6 -E07000074,15,7 -E07000075,15,6 -E07000076,16,8 -E07000077,13,7 -E07000078,1,5 -E07000079,2,5 -E07000080,-1,6 -E07000081,0,6 -E07000082,0,5 -E07000083,1,6 -E07000084,2,1 -E07000085,4,0 -E07000086,3,0 -E07000087,2,-1 -E07000088,3,-1 -E07000089,3,2 -E07000090,5,0 -E07000091,1,0 -E07000092,4,1 -E07000093,1,1 -E07000094,3,1 -E07000095,12,6 -E07000096,8,6 -E07000098,9,5 -E07000099,11,7 -E07000102,7,5 -E07000103,8,5 -E07000105,12,-1 -E07000106,15,0 -E07000107,13,1 -E07000108,14,-1 -E07000109,14,1 -E07000110,14,0 -E07000111,12,0 -E07000112,13,-1 -E07000113,16,0 -E07000114,15,-1 -E07000115,13,0 -E07000116,11,-1 -E07000117,6,15 -E07000118,3,14 -E07000119,4,16 -E07000120,5,15 -E07000121,3,17 -E07000122,6,16 -E07000123,5,16 -E07000124,5,17 -E07000125,6,14 -E07000126,3,15 -E07000127,2,13 -E07000128,3,16 -E07000129,7,7 -E07000130,8,9 -E07000131,8,7 -E07000132,7,8 -E07000133,11,10 -E07000134,6,9 -E07000135,9,8 -E07000136,12,12 -E07000137,12,13 -E07000138,11,12 -E07000139,11,11 -E07000140,12,11 -E07000141,12,10 -E07000142,11,13 -E07000143,14,10 -E07000144,15,12 -E07000145,15,11 -E07000146,13,11 -E07000147,14,12 -E07000148,14,11 -E07000149,15,10 -E07000170,8,11 -E07000171,10,13 -E07000172,7,10 -E07000173,9,10 -E07000174,9,11 -E07000175,10,11 -E07000176,10,10 -E07000177,4,5 -E07000178,4,4 -E07000179,5,4 -E07000180,3,4 -E07000181,3,5 -E07000192,3,9 -E07000193,5,11 -E07000194,4,9 -E07000195,2,10 -E07000196,2,8 -E07000197,4,10 -E07000198,5,10 -E07000199,5,9 -E07000200,14,8 -E07000202,15,9 -E07000203,14,9 -E07000207,7,2 -E07000208,8,0 -E07000209,5,1 -E07000210,6,1 -E07000211,7,0 -E07000212,5,3 -E07000213,6,3 -E07000214,5,2 -E07000215,9,-1 -E07000216,6,0 -E07000217,6,2 -E07000218,6,8 -E07000219,6,7 -E07000220,6,6 -E07000221,3,6 -E07000222,4,6 -E07000223,8,-2 -E07000224,6,-2 -E07000225,5,-1 -E07000226,8,-1 -E07000227,6,-1 -E07000228,7,-1 -E07000229,7,-2 -E07000234,2,7 -E07000235,-1,7 -E07000236,4,7 -E07000237,0,7 -E07000238,2,6 -E07000239,1,8 -E07000240,10,6 -E07000241,11,6 -E07000242,13,8 -E07000243,12,7 -E07000244,16,10 -E07000245,13,9 -E08000001,4,14 -E08000002,5,14 -E08000003,5,12 -E08000004,5,13 -E08000005,7,14 -E08000006,4,13 -E08000007,6,12 -E08000008,6,13 -E08000009,4,12 -E08000010,3,13 -E08000011,2,12 -E08000012,1,13 -E08000013,3,12 -E08000014,2,14 -E08000015,1,12 -E08000016,8,14 -E08000017,9,14 -E08000018,9,13 -E08000019,8,13 -E08000021,5,19 -E08000022,6,20 -E08000023,7,20 -E08000024,7,19 -E08000025,5,8 -E08000026,5,6 -E08000027,1,7 -E08000028,3,7 -E08000029,5,7 -E08000030,4,8 -E08000031,3,8 -E08000032,7,16 -E08000033,7,15 -E08000034,8,15 -E08000035,8,16 -E08000036,9,15 -E08000037,6,19 -E09000001,11,2 -E09000002,13,3 -E09000003,10,5 -E09000004,12,1 -E09000005,10,4 -E09000006,11,0 -E09000007,11,4 -E09000008,10,0 -E09000009,9,4 -E09000010,11,5 -E09000011,11,1 -E09000012,12,3 -E09000013,8,3 -E09000014,12,4 -E09000015,8,4 -E09000016,14,3 -E09000017,7,4 -E09000018,7,3 -E09000019,11,3 -E09000020,9,3 -E09000021,7,1 -E09000022,10,2 -E09000023,10,1 -E09000024,8,1 -E09000025,13,2 -E09000026,14,4 -E09000027,8,2 -E09000028,9,1 -E09000029,9,0 -E09000030,12,2 -E09000031,13,4 -E09000032,9,2 -E09000033,10,3 -N09000001,-4,16 -N09000002,-5,16 -N09000003,-4,17 -N09000004,-5,18 -N09000005,-6,17 -N09000006,-6,16 -N09000007,-5,15 -N09000008,-4,18 -N09000009,-5,17 -N09000010,-4,15 -N09000011,-3,16 -S12000005,2,24 -S12000006,4,20 -S12000008,3,20 -S12000010,5,22 -S12000011,2,20 -S12000013,-1,27 -S12000014,2,23 -S12000017,1,26 -S12000018,0,21 -S12000019,3,21 -S12000020,2,26 -S12000021,1,20 -S12000023,4,28 -S12000026,4,21 -S12000027,5,30 -S12000028,1,19 -S12000029,2,21 -S12000030,1,24 -S12000033,4,26 -S12000034,3,26 -S12000035,0,24 -S12000036,4,22 -S12000038,1,22 -S12000039,0,23 -S12000040,3,22 -S12000041,2,25 -S12000042,3,25 -S12000045,1,23 -S12000047,3,24 -S12000048,1,25 -S12000049,1,21 -S12000050,2,22 -W06000001,-2,12 -W06000002,-2,10 -W06000003,-1,10 -W06000004,0,10 -W06000005,0,11 -W06000006,1,10 -W06000008,-2,9 -W06000009,-5,6 -W06000010,-4,6 -W06000011,-4,5 -W06000012,-3,5 -W06000013,-3,6 -W06000014,-2,4 -W06000015,-2,5 -W06000016,-3,7 -W06000018,-2,6 -W06000019,0,9 -W06000020,-2,7 -W06000021,-1,8 -W06000022,-1,5 -W06000023,-1,9 -W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py index 18194145d..70766120a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py @@ -76,47 +76,24 @@ class LocalAuthorityBoundaryDefinition: ) -@dataclass(frozen=True, slots=True) -class LocalAuthorityCoordinate: - """Integer rendering coordinate for one local authority.""" - - x: int - y: int - - -@dataclass(frozen=True, slots=True) -class LocalAuthorityDisplayMetadata: - """Display-only metadata attached after authority impacts are calculated.""" - - name: str - x: int - y: int - - @dataclass(frozen=True, slots=True) class UKLocalAuthorityResources: - """Validated display names and coordinates for supported boundary versions.""" + """Validated display names and each supported boundary version's codes. + + ``names`` maps every supported authority code to its display name, and + ``codes`` lists the authority codes of each boundary version. + """ names: Mapping[str, str] - coordinates: Mapping[ - UKLocalAuthorityBoundaryVersion, - Mapping[str, LocalAuthorityCoordinate], - ] + codes: Mapping[UKLocalAuthorityBoundaryVersion, frozenset[str]] - def metadata_for( + def names_for( self, boundary_version: UKLocalAuthorityBoundaryVersion, - ) -> Mapping[str, LocalAuthorityDisplayMetadata]: - """Combine common names with one boundary version's rendering coordinates.""" + ) -> Mapping[str, str]: + """Return the display name of each authority in one boundary version.""" return MappingProxyType( - { - code: LocalAuthorityDisplayMetadata( - name=self.names[code], - x=coordinate.x, - y=coordinate.y, - ) - for code, coordinate in self.coordinates[boundary_version].items() - } + {code: self.names[code] for code in sorted(self.codes[boundary_version])} ) @@ -167,55 +144,39 @@ def _load_names() -> Mapping[str, str]: return MappingProxyType(names) -def _load_coordinates(filename: str) -> Mapping[str, LocalAuthorityCoordinate]: - coordinates: dict[str, LocalAuthorityCoordinate] = {} - for row in _rows(filename, ("code", "x", "y")): +def _load_codes(filename: str) -> frozenset[str]: + codes: set[str] = set() + for row in _rows(filename, ("code",)): code = row["code"] _require_code(code, filename) - if code in coordinates: + if code in codes: msg = f"UK local-authority resource {filename!r} repeats {code!r}" raise ValueError(msg) - try: - coordinate = LocalAuthorityCoordinate(x=int(row["x"]), y=int(row["y"])) - except ValueError as error: - msg = ( - f"UK local-authority resource {filename!r} has invalid coordinates " - f"for {code!r}" - ) - raise ValueError(msg) from error - coordinates[code] = coordinate - return MappingProxyType(coordinates) + codes.add(code) + return frozenset(codes) @lru_cache(maxsize=1) def load_uk_local_authority_resources() -> UKLocalAuthorityResources: """Load and validate every packaged local-authority resource.""" names = _load_names() - lad22 = _load_coordinates("coordinates_lad22.csv") - lad23 = _load_coordinates("coordinates_lad23.csv") - supported_codes = lad22.keys() | lad23.keys() - if names.keys() != supported_codes: + lad22 = _load_codes("codes_lad22.csv") + lad23 = _load_codes("codes_lad23.csv") + if names.keys() != lad22 | lad23: msg = ( "UK local-authority names must exactly cover the supported " "boundary versions" ) raise ValueError(msg) - if lad22.keys() - lad23.keys() != _LAD22_ONLY_CODES: - msg = "LAD22 distinguishing codes do not match its coordinate file" + if lad22 - lad23 != _LAD22_ONLY_CODES: + msg = "LAD22 distinguishing codes do not match its code file" raise ValueError(msg) - if lad23.keys() - lad22.keys() != _LAD23_ONLY_CODES: - msg = "LAD23 distinguishing codes do not match its coordinate file" + if lad23 - lad22 != _LAD23_ONLY_CODES: + msg = "LAD23 distinguishing codes do not match its code file" raise ValueError(msg) - for code in lad22.keys() & lad23.keys(): - if lad22[code] != lad23[code]: - msg = ( - "UK local-authority coordinate differs between boundary versions " - f"for {code!r}" - ) - raise ValueError(msg) return UKLocalAuthorityResources( names=names, - coordinates=MappingProxyType( + codes=MappingProxyType( { UKLocalAuthorityBoundaryVersion.LAD22: lad22, UKLocalAuthorityBoundaryVersion.LAD23: lad23, diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index ccf4a1918..41d416d4f 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -213,8 +213,8 @@ def test_v2_static_runtime_file_smoke_uses_exact_uk_worker_image(monkeypatch) -> ) assert smoke.smoke_stage12_static_runtime_files() == { "names": 378, - "lad22_coordinates": 374, - "lad23_coordinates": 361, + "lad22_codes": 374, + "lad23_codes": 361, "installed_dataset_boundary_version": "lad22", } diff --git a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py index 20552792b..853a3f6da 100644 --- a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py @@ -2,13 +2,19 @@ from __future__ import annotations +import hashlib +import re +import shutil +from importlib.resources import files from types import SimpleNamespace +from typing import TYPE_CHECKING import pandas as pd import pytest from policyengine_simulation_contract.uk_geography import ( UKLocalAuthorityBoundaryVersion, ) +from policyengine_simulation_executor import uk_local_authority_metadata from policyengine_simulation_executor.uk_local_authority_metadata import ( detect_uk_area_dataset_metadata, detect_uk_local_authority_boundary_version, @@ -22,6 +28,10 @@ uk_hdf_household_has_local_authority_codes, ) +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from pathlib import Path + LAD22_ONLY_CODES = { "E07000026", "E07000027", @@ -49,42 +59,140 @@ } +# SHA-256 of each boundary version's sorted codes, one per line, as listed by +# coordinates_lad22.csv and coordinates_lad23.csv before the executor stopped +# shipping local-authority map positions. The code lists that replaced those +# files must keep exactly these codes. +RETIRED_COORDINATE_FILE_CODE_DIGESTS = { + UKLocalAuthorityBoundaryVersion.LAD22: ( + "36b51d2d163e3e094eebd838daf3417a416ded29cd4d3b54af4901ae9c58e37f" + ), + UKLocalAuthorityBoundaryVersion.LAD23: ( + "9a313a86a8b4542d4020cc50d613fcce8d09dd806253f1716ff3e126a8ed1ab2" + ), +} +PACKAGED_RESOURCE_DIRECTORY = files("policyengine_simulation_executor").joinpath( + "static_runtime_files", "uk_local_authorities" +) + + def test_packaged_resources_cover_the_exact_supported_boundary_versions() -> None: resources = load_uk_local_authority_resources() - lad22 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - lad23 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] + lad22 = resources.codes[UKLocalAuthorityBoundaryVersion.LAD22] + lad23 = resources.codes[UKLocalAuthorityBoundaryVersion.LAD23] assert len(resources.names) == 378 assert len(lad22) == 374 assert len(lad23) == 361 - assert len(lad22.keys() & lad23.keys()) == 357 - assert lad22.keys() - lad23.keys() == LAD22_ONLY_CODES - assert lad23.keys() - lad22.keys() == LAD23_ONLY_CODES - assert resources.names.keys() == lad22.keys() | lad23.keys() + assert len(lad22 & lad23) == 357 + assert lad22 - lad23 == LAD22_ONLY_CODES + assert lad23 - lad22 == LAD23_ONLY_CODES + assert resources.names.keys() == lad22 | lad23 + + +@pytest.mark.parametrize( + ("boundary_version", "digest"), RETIRED_COORDINATE_FILE_CODE_DIGESTS.items() +) +def test_each_boundary_version_keeps_the_codes_its_coordinate_file_listed( + boundary_version: UKLocalAuthorityBoundaryVersion, digest: str +) -> None: + codes = load_uk_local_authority_resources().codes[boundary_version] + listed = "\n".join(sorted(codes)).encode() + assert hashlib.sha256(listed).hexdigest() == digest -def test_shared_authorities_have_identical_coordinates() -> None: + +def test_the_executor_ships_names_and_code_lists_without_map_positions() -> None: + packaged = sorted(entry.name for entry in PACKAGED_RESOURCE_DIRECTORY.iterdir()) + + assert packaged == ["codes_lad22.csv", "codes_lad23.csv", "names.csv"] + + +def test_names_for_a_boundary_version_cover_exactly_its_codes() -> None: resources = load_uk_local_authority_resources() - lad22 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - lad23 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] - for code in lad22.keys() & lad23.keys(): - assert lad22[code] == lad23[code] + for boundary_version in UKLocalAuthorityBoundaryVersion: + names = resources.names_for(boundary_version) + assert names.keys() == resources.codes[boundary_version] + lad23_names = resources.names_for(UKLocalAuthorityBoundaryVersion.LAD23) + assert lad23_names["E06000063"] == "Cumberland" + assert "E06000063" not in resources.names_for(UKLocalAuthorityBoundaryVersion.LAD22) + + +@pytest.fixture +def editable_resources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Path]: + """Load the packaged resources from copies a test may change.""" + for entry in PACKAGED_RESOURCE_DIRECTORY.iterdir(): + with entry.open("rb") as source, (tmp_path / entry.name).open("wb") as copy: + shutil.copyfileobj(source, copy) + monkeypatch.setattr( + uk_local_authority_metadata, "_resource_path", lambda name: tmp_path / name + ) + load_uk_local_authority_resources.cache_clear() + yield tmp_path + load_uk_local_authority_resources.cache_clear() + +def _rewrite_lines(path: Path, change: Callable[[list[str]], list[str]]) -> None: + lines = path.read_text(encoding="utf-8").splitlines() + path.write_text("\n".join(change(lines)) + "\n", encoding="utf-8") -def test_resource_lookup_combines_names_with_boundary_coordinates() -> None: + +def test_resources_load_from_unchanged_copies(editable_resources: Path) -> None: resources = load_uk_local_authority_resources() - cumberland = resources.metadata_for(UKLocalAuthorityBoundaryVersion.LAD23)[ - "E06000063" - ] - assert cumberland.name == "Cumberland" - assert isinstance(cumberland.x, int) - assert isinstance(cumberland.y, int) - assert "E06000063" not in resources.metadata_for( - UKLocalAuthorityBoundaryVersion.LAD22 + assert len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD22]) == 374 + + +def test_resources_refuse_names_that_miss_a_supported_code( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "names.csv", + lambda lines: [line for line in lines if not line.startswith("E06000063,")], + ) + + with pytest.raises(ValueError, match="must exactly cover"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_code_list_with_other_distinguishing_codes( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "codes_lad23.csv", + # E07000026 is one of the codes only LAD22 has. + lambda lines: [lines[0], *sorted([*lines[1:], "E07000026"])], ) + with pytest.raises(ValueError, match="LAD22 distinguishing codes"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_repeated_code(editable_resources: Path) -> None: + _rewrite_lines( + editable_resources / "codes_lad22.csv", lambda lines: [*lines, lines[1]] + ) + + with pytest.raises(ValueError, match="repeats 'E06000001'"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_code_list_that_carries_map_positions( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "codes_lad22.csv", + lambda lines: ["code,x,y", *(f"{code},0,0" for code in lines[1:])], + ) + + with pytest.raises( + ValueError, match=re.escape("'codes_lad22.csv' has invalid columns") + ): + load_uk_local_authority_resources() + def test_detector_identifies_lad22_from_a_predecessor_code() -> None: metadata = detect_uk_local_authority_boundary_version( From fdabde000985c8a31a43733f8a4ff40962a6b652 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:12:18 +0400 Subject: [PATCH 4/5] Reword the migration doc's map-position sentence It said map positions are "now" drawn by the front end, which depends on an app change that has not shipped. Say that results carry no map positions and that the app draws them from its own layout files. Docs only; no behaviour changes, so there is no test. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-12-modal-worker-foundation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 347e1f334..ff2c1bf27 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -286,8 +286,8 @@ use the existing GCS lookup and are not changed by this path. This resource ownership is temporary. A future boundary version must add its codes to the common names file, add a distinct code file, and register a typed -detector definition. Map positions are now the front end's to draw from its -own layout files, and Stage 14 or later work should remove these temporary +detector definition. Results carry no map positions, which the app draws from +its own layout files, and Stage 14 or later work should remove these temporary executor resources when the authoritative v2 report architecture supersedes Stage 12. From b54712ce6aabdd04b683ba62c38ace53a2248f33 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:37:52 +0400 Subject: [PATCH 5/5] Drop the older-app wording now that the contract change is its own PR Requests never reach apps built for older policyengine.py versions, and #985 (fixes #984) already made x and y optional in the contract, so: - The changelog fragment keeps only this branch's executor change and code lists; the contract paragraph is #984's fragment. - The breakdown check's docstring says the executor no longer sends x or y. - The generated-client test that read records with and without x and y now covers only records with them, from breakdowns that started before this deploy; the case without them is #985's test. Co-Authored-By: Claude Opus 5.5 --- changelog.d/918.changed.md | 4 +--- .../tests/simulation/test_calculate.py | 22 ++++++------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/changelog.d/918.changed.md b/changelog.d/918.changed.md index b5d51ab0f..bc05d6f9f 100644 --- a/changelog.d/918.changed.md +++ b/changelog.d/918.changed.md @@ -1,5 +1,3 @@ -The `x` and `y` fields of UK constituency and local-authority records, each area's integer position on the hex map, are now optional in the contract. A record validates without them, and its JSON leaves them out instead of writing `null`. Records from apps deployed for older policyengine.py versions, which still send both, validate and keep them. The gateway's OpenAPI document no longer lists them as required, and the contract now requires pydantic 2.12 or later, the first release with field-level `exclude_if`. - -The executor no longer sends `x` or `y` on UK constituency and local-authority records. It no longer reads the `x` and `y` columns of policyengine.py's constituency lookup or attaches its packaged local-authority positions, and it no longer refuses a published area without a map position. Results from apps deployed from now on carry neither key, while apps deployed for older policyengine.py versions keep sending both. The beta integration check of a local-area breakdown now requires records without them. +The executor no longer sends `x` or `y` on UK constituency and local-authority records. It no longer reads the `x` and `y` columns of policyengine.py's constituency lookup or attaches its packaged local-authority positions, and it no longer refuses a published area without a map position. Results it returns from now on carry neither key, which the contract already allows (#984). The beta integration check of a local-area breakdown now requires records without them. The executor no longer ships `coordinates_lad22.csv` and `coordinates_lad23.csv`. Each boundary version's local-authority codes now come from `codes_lad22.csv` and `codes_lad23.csv`, which list exactly the codes the coordinate files did, so detecting LAD22 or LAD23 codes, and its checks, are unchanged. The Stage 12 static runtime file smoke test reports `lad22_codes` and `lad23_codes` instead of `lad22_coordinates` and `lad23_coordinates`. diff --git a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py index f049886dc..b4dcae7b9 100644 --- a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py +++ b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py @@ -263,8 +263,8 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: """Check a local-area breakdown came from the local-area dataset. Both area levels must publish at least one area, and every published area - must carry a disclosure tier and no x or y, which only apps deployed for - older policyengine.py versions still send. + must carry a disclosure tier and no x or y, which the executor no longer + sends. """ breakdown = _as_mapping(breakdown_result, label="local-area breakdown") assert breakdown.get("kind") == "local_areas", ( @@ -390,21 +390,13 @@ def test_generated_client_reads_area_records_without_x_and_y() -> None: assert status.result.to_dict() == breakdown -@pytest.mark.parametrize( - "map_position", - [ - pytest.param({}, id="current-app"), - pytest.param({"x": 56, "y": -40}, id="older-app"), - ], -) -def test_generated_client_reads_area_records_with_and_without_x_and_y( - map_position: dict[str, int], -) -> None: - # Apps deployed for older policyengine.py versions still send x and y. +def test_generated_client_reads_area_records_with_x_and_y() -> None: + # Breakdowns that started before the executor stopped sending x and y + # still carry them, until the contract removes the two fields. breakdown = { **_BREAKDOWN_RESULT, - "constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, **map_position}], - "local_authority_impact": [{**_BREAKDOWN_LOCAL_AUTHORITY, **map_position}], + "constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}], + "local_authority_impact": [{**_BREAKDOWN_LOCAL_AUTHORITY, "x": 4, "y": 18}], } status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown})