diff --git a/changelog.d/918.changed.md b/changelog.d/918.changed.md new file mode 100644 index 000000000..bc05d6f9f --- /dev/null +++ b/changelog.d/918.changed.md @@ -0,0 +1,3 @@ +The executor no longer sends `x` or `y` on UK constituency and local-authority records. It no longer reads the `x` and `y` columns of policyengine.py's constituency lookup or attaches its packaged local-authority positions, and it no longer refuses a published area without a map position. Results it returns from now on carry neither key, which the contract already allows (#984). The beta integration check of a local-area breakdown now requires records without them. + +The executor no longer ships `coordinates_lad22.csv` and `coordinates_lad23.csv`. Each boundary version's local-authority codes now come from `codes_lad22.csv` and `codes_lad23.csv`, which list exactly the codes the coordinate files did, so detecting LAD22 or LAD23 codes, and its checks, are unchanged. The Stage 12 static runtime file smoke test reports `lad22_codes` and `lad23_codes` instead of `lad22_coordinates` and `lad23_coordinates`. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 4ecb5fc0e..ff2c1bf27 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -255,21 +255,29 @@ configuration, an unknown code, or a dataset without a distinguishing code fails the Stage 12 run. Dataset names and release labels do not select a boundary version, and there is no fallback to the newest known configuration. -The executor temporarily packages three display-only resources under its -expandable `static_runtime_files/uk_local_authorities/` directory. The v2 Modal -image copies that directory explicitly because Modal's Python-source layer does -not include non-Python files by default: - -- one common `code,name` file containing the union of supported authority - codes; -- one `code,x,y` file for the 374-authority LAD22 configuration; and -- one `code,x,y` file for the 361-authority LAD23 configuration. +The executor temporarily packages three resources under its expandable +`static_runtime_files/uk_local_authorities/` directory. The v2 Modal image +copies that directory explicitly because Modal's Python-source layer does not +include non-Python files by default: + +- one common `code,name` file, `names.csv`, containing the union of supported + authority codes; +- one `code` file, `codes_lad22.csv`, listing the 374-authority LAD22 + configuration; and +- one `code` file, `codes_lad23.csv`, listing the 361-authority LAD23 + configuration. + +The two code lists replaced `coordinates_lad22.csv` and +`coordinates_lad23.csv`, which also gave each authority an integer map +position (`x` and `y`). Results no longer carry a map position, so the +executor no longer ships one; the code lists keep exactly the codes those +files listed. The Enhanced FRS geography is LAD22: its 2021 output areas are crosswalked to 2022 local-authority boundaries. The output-area vintage does not make the -local-authority boundary version LAD21. Names and coordinates are attached only -after numeric local-authority impacts have been calculated; these resources -never assign a household to an authority. +local-authority boundary version LAD21. Names are attached only after numeric +local-authority impacts have been calculated; these resources never assign a +household to an authority. The detected boundary version is stored as typed metadata in both child Parquet artifacts. The coordinator requires baseline and reform to agree before it @@ -277,9 +285,9 @@ enriches the aggregate output. Existing non-Stage-12 calculations continue to use the existing GCS lookup and are not changed by this path. This resource ownership is temporary. A future boundary version must add its -codes to the common names file, add a distinct coordinate file, and register a -typed detector definition. Longer term, display-coordinate rendering should -move to the front end, and Stage 14 or later work should remove these temporary +codes to the common names file, add a distinct code file, and register a typed +detector definition. Results carry no map positions, which the app draws from +its own layout files, and Stage 14 or later work should remove these temporary executor resources when the authoritative v2 report architecture supersedes Stage 12. diff --git a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py index 10c659678..b4dcae7b9 100644 --- a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py +++ b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py @@ -263,7 +263,8 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: """Check a local-area breakdown came from the local-area dataset. Both area levels must publish at least one area, and every published area - must carry a disclosure tier. + must carry a disclosure tier and no x or y, which the executor no longer + sends. """ breakdown = _as_mapping(breakdown_result, label="local-area breakdown") assert breakdown.get("kind") == "local_areas", ( @@ -283,6 +284,10 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: assert record.get("disclosure_tier") in _PUBLISHED_DISCLOSURE_TIERS, ( f"Expected a published disclosure_tier in {key}, got {record}" ) + map_position = {"x", "y"} & set(record) + assert not map_position, ( + f"Expected {key} records without x or y, got {record}" + ) class _GeneratedClientObject: @@ -326,8 +331,6 @@ def test_result_assertions_accept_generated_client_objects() -> None: _BREAKDOWN_CONSTITUENCY = { "constituency_code": "E14001063", "constituency_name": "Aldershot", - "x": 56, - "y": -40, "average_household_income_change": -150.0, "relative_household_income_change": -0.0026, "population": 40530.0, @@ -340,8 +343,6 @@ def test_result_assertions_accept_generated_client_objects() -> None: _BREAKDOWN_LOCAL_AUTHORITY = { "local_authority_code": "E06000063", "local_authority_name": "Cumberland", - "x": 4, - "y": 18, "average_household_income_change": -120.0, "relative_household_income_change": -0.002, "population": 120000.0, @@ -389,6 +390,21 @@ def test_generated_client_reads_area_records_without_x_and_y() -> None: assert status.result.to_dict() == breakdown +def test_generated_client_reads_area_records_with_x_and_y() -> None: + # Breakdowns that started before the executor stopped sending x and y + # still carry them, until the contract removes the two fields. + breakdown = { + **_BREAKDOWN_RESULT, + "constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}], + "local_authority_impact": [{**_BREAKDOWN_LOCAL_AUTHORITY, "x": 4, "y": 18}], + } + + status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown}) + + assert isinstance(status.result, LocalAreaBreakdownOutput) + assert status.result.to_dict() == breakdown + + @pytest.mark.parametrize( ("change", "message"), [ @@ -403,6 +419,18 @@ def test_generated_client_reads_area_records_without_x_and_y() -> None: }, "disclosure_tier", ), + ( + {"constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}]}, + "x or y", + ), + ( + { + "local_authority_impact": [ + {**_BREAKDOWN_LOCAL_AUTHORITY, "x": 4, "y": 18} + ] + }, + "x or y", + ), ], ) def test_breakdown_assertions_reject_an_incomplete_breakdown(change, message) -> None: diff --git a/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py b/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py index 791f99e15..599e466ae 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_static_runtime_files_smoke.py @@ -44,12 +44,8 @@ def smoke_stage12_static_runtime_files() -> dict[str, int | str]: resources = load_uk_local_authority_resources() return { "names": len(resources.names), - "lad22_coordinates": len( - resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - ), - "lad23_coordinates": len( - resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] - ), + "lad22_codes": len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD22]), + "lad23_codes": len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD23]), "installed_dataset_boundary_version": ( validate_installed_uk_local_authority_dataset() ), @@ -62,8 +58,8 @@ def main() -> None: observed = smoke_stage12_static_runtime_files.remote() expected_counts = { "names": 378, - "lad22_coordinates": 374, - "lad23_coordinates": 361, + "lad22_codes": 374, + "lad23_codes": 361, } if {key: observed.get(key) for key in expected_counts} != expected_counts: msg = ( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 4ce4a49b8..e79339525 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import math from collections.abc import Mapping from dataclasses import dataclass, field from functools import lru_cache @@ -82,7 +81,7 @@ def _complete_uk_geography_output( code_field: str, name_field: str, ) -> GeographicImpactOutput: - """Require every UK geography record to contain lookup-owned metadata. + """Require every UK geography record to carry its code and lookup name. An empty list is valid: disclosure control may withhold every area. """ @@ -100,9 +99,6 @@ def _complete_uk_geography_output( if not isinstance(name, str) or not name.strip() or name == code: msg = f"UK geography {code!r} is missing a lookup name" raise ValueError(msg) - if record_values.get("x") is None or record_values.get("y") is None: - msg = f"UK geography {code!r} is missing lookup coordinates" - raise ValueError(msg) return output @@ -646,10 +642,12 @@ def compute_uk_geography_disclosure_output( ) -def _uk_constituency_lookup( - lookup_csv_path: str, -) -> dict[str, tuple[str, int | None, int | None]]: - """Return each constituency's name and map coordinates, in lookup order.""" +def _uk_constituency_lookup(lookup_csv_path: str) -> dict[str, str]: + """Return each constituency's name, in lookup order. + + The lookup's other columns, such as its x and y map positions, are not + read: sim-api results no longer carry an area's map position. + """ lookup = pd.read_csv(lookup_csv_path) if "code" not in lookup.columns: msg = ( @@ -657,28 +655,13 @@ def _uk_constituency_lookup( f"{lookup_csv_path}" ) raise ValueError(msg) - entries: dict[str, tuple[str, int | None, int | None]] = {} + names: dict[str, str] = {} for row in lookup.to_dict("records"): code = _normalised_area_code(row.get("code")) if not code: continue - name = _normalised_area_code(row.get("name")) or code - entries[code] = ( - name, - _lookup_coordinate(row.get("x")), - _lookup_coordinate(row.get("y")), - ) - return entries - - -def _lookup_coordinate(value: object) -> int | None: - if isinstance(value, bool) or not isinstance(value, int | float | str): - return None - try: - number = float(value) - except ValueError: - return None - return int(number) if math.isfinite(number) else None + names[code] = _normalised_area_code(row.get("name")) or code + return names def _require_uk_disclosure_output( @@ -699,7 +682,7 @@ def build_uk_constituency_impact( Areas that disclosure control withholds are left out. Downloads the constituency lookup if needed and raises when a published area lacks a - lookup name or coordinates. + lookup name. """ if country != "uk": return None @@ -712,18 +695,14 @@ def build_uk_constituency_impact( ordered = [code for code in lookup if code in changes] + sorted( set(changes) - set(lookup) ) - records: list[dict[str, object]] = [] - for code in ordered: - name, x, y = lookup.get(code, (code, None, None)) - records.append( - { - "constituency_code": code, - "constituency_name": name, - "x": x, - "y": y, - **changes[code].record_fields(), - } - ) + records: list[dict[str, object]] = [ + { + "constituency_code": code, + "constituency_name": lookup.get(code, code), + **changes[code].record_fields(), + } + for code in ordered + ] return _complete_uk_geography_output( records, code_field="constituency_code", @@ -737,7 +716,7 @@ def build_uk_local_authority_impact( *, uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> GeographicImpactOutput | None: - """Return the published UK local authorities, named and placed for the boundaries. + """Return the published UK local authorities, named for the boundary version. None for other countries. For the UK, ``uk_local_authority_metadata`` is required, and a code outside its boundary version raises ``ValueError``. @@ -757,13 +736,11 @@ def build_uk_local_authority_impact( ) boundary_version = uk_local_authority_metadata.boundary_version - display_metadata = load_uk_local_authority_resources().metadata_for( - boundary_version - ) + names = load_uk_local_authority_resources().names_for(boundary_version) records: list[dict[str, object]] = [] for code in sorted(changes): - metadata = display_metadata.get(code) - if metadata is None: + name = names.get(code) + if name is None: msg = ( f"UK local-authority code {code!r} is not part of the detected " f"{boundary_version.value.upper()} boundary version" @@ -772,9 +749,7 @@ def build_uk_local_authority_impact( records.append( { "local_authority_code": code, - "local_authority_name": metadata.name, - "x": metadata.x, - "y": metadata.y, + "local_authority_name": name, **changes[code].record_fields(), } ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py index 9ff959eec..67361697a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py @@ -122,8 +122,8 @@ def _check_uk_local_authority_resources() -> None: ) resources = load_uk_local_authority_resources() - for boundary_version in resources.coordinates: - resources.metadata_for(boundary_version) + for boundary_version in resources.codes: + resources.names_for(boundary_version) def _check_uk_local_authority_dataset(dataset_path: str) -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv new file mode 100644 index 000000000..111f7f79d --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad22.csv @@ -0,0 +1,375 @@ +code +E06000001 +E06000002 +E06000003 +E06000004 +E06000005 +E06000006 +E06000007 +E06000008 +E06000009 +E06000010 +E06000011 +E06000012 +E06000013 +E06000014 +E06000015 +E06000016 +E06000017 +E06000018 +E06000019 +E06000020 +E06000021 +E06000022 +E06000023 +E06000024 +E06000025 +E06000026 +E06000027 +E06000030 +E06000031 +E06000032 +E06000033 +E06000034 +E06000035 +E06000036 +E06000037 +E06000038 +E06000039 +E06000040 +E06000041 +E06000042 +E06000043 +E06000044 +E06000045 +E06000046 +E06000047 +E06000049 +E06000050 +E06000051 +E06000052 +E06000053 +E06000054 +E06000055 +E06000056 +E06000057 +E06000058 +E06000059 +E06000060 +E06000061 +E06000062 +E07000008 +E07000009 +E07000010 +E07000011 +E07000012 +E07000026 +E07000027 +E07000028 +E07000029 +E07000030 +E07000031 +E07000032 +E07000033 +E07000034 +E07000035 +E07000036 +E07000037 +E07000038 +E07000039 +E07000040 +E07000041 +E07000042 +E07000043 +E07000044 +E07000045 +E07000046 +E07000047 +E07000061 +E07000062 +E07000063 +E07000064 +E07000065 +E07000066 +E07000067 +E07000068 +E07000069 +E07000070 +E07000071 +E07000072 +E07000073 +E07000074 +E07000075 +E07000076 +E07000077 +E07000078 +E07000079 +E07000080 +E07000081 +E07000082 +E07000083 +E07000084 +E07000085 +E07000086 +E07000087 +E07000088 +E07000089 +E07000090 +E07000091 +E07000092 +E07000093 +E07000094 +E07000095 +E07000096 +E07000098 +E07000099 +E07000102 +E07000103 +E07000105 +E07000106 +E07000107 +E07000108 +E07000109 +E07000110 +E07000111 +E07000112 +E07000113 +E07000114 +E07000115 +E07000116 +E07000117 +E07000118 +E07000119 +E07000120 +E07000121 +E07000122 +E07000123 +E07000124 +E07000125 +E07000126 +E07000127 +E07000128 +E07000129 +E07000130 +E07000131 +E07000132 +E07000133 +E07000134 +E07000135 +E07000136 +E07000137 +E07000138 +E07000139 +E07000140 +E07000141 +E07000142 +E07000143 +E07000144 +E07000145 +E07000146 +E07000147 +E07000148 +E07000149 +E07000163 +E07000164 +E07000165 +E07000166 +E07000167 +E07000168 +E07000169 +E07000170 +E07000171 +E07000172 +E07000173 +E07000174 +E07000175 +E07000176 +E07000177 +E07000178 +E07000179 +E07000180 +E07000181 +E07000187 +E07000188 +E07000189 +E07000192 +E07000193 +E07000194 +E07000195 +E07000196 +E07000197 +E07000198 +E07000199 +E07000200 +E07000202 +E07000203 +E07000207 +E07000208 +E07000209 +E07000210 +E07000211 +E07000212 +E07000213 +E07000214 +E07000215 +E07000216 +E07000217 +E07000218 +E07000219 +E07000220 +E07000221 +E07000222 +E07000223 +E07000224 +E07000225 +E07000226 +E07000227 +E07000228 +E07000229 +E07000234 +E07000235 +E07000236 +E07000237 +E07000238 +E07000239 +E07000240 +E07000241 +E07000242 +E07000243 +E07000244 +E07000245 +E07000246 +E08000001 +E08000002 +E08000003 +E08000004 +E08000005 +E08000006 +E08000007 +E08000008 +E08000009 +E08000010 +E08000011 +E08000012 +E08000013 +E08000014 +E08000015 +E08000016 +E08000017 +E08000018 +E08000019 +E08000021 +E08000022 +E08000023 +E08000024 +E08000025 +E08000026 +E08000027 +E08000028 +E08000029 +E08000030 +E08000031 +E08000032 +E08000033 +E08000034 +E08000035 +E08000036 +E08000037 +E09000001 +E09000002 +E09000003 +E09000004 +E09000005 +E09000006 +E09000007 +E09000008 +E09000009 +E09000010 +E09000011 +E09000012 +E09000013 +E09000014 +E09000015 +E09000016 +E09000017 +E09000018 +E09000019 +E09000020 +E09000021 +E09000022 +E09000023 +E09000024 +E09000025 +E09000026 +E09000027 +E09000028 +E09000029 +E09000030 +E09000031 +E09000032 +E09000033 +N09000001 +N09000002 +N09000003 +N09000004 +N09000005 +N09000006 +N09000007 +N09000008 +N09000009 +N09000010 +N09000011 +S12000005 +S12000006 +S12000008 +S12000010 +S12000011 +S12000013 +S12000014 +S12000017 +S12000018 +S12000019 +S12000020 +S12000021 +S12000023 +S12000026 +S12000027 +S12000028 +S12000029 +S12000030 +S12000033 +S12000034 +S12000035 +S12000036 +S12000038 +S12000039 +S12000040 +S12000041 +S12000042 +S12000045 +S12000047 +S12000048 +S12000049 +S12000050 +W06000001 +W06000002 +W06000003 +W06000004 +W06000005 +W06000006 +W06000008 +W06000009 +W06000010 +W06000011 +W06000012 +W06000013 +W06000014 +W06000015 +W06000016 +W06000018 +W06000019 +W06000020 +W06000021 +W06000022 +W06000023 +W06000024 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv new file mode 100644 index 000000000..f184d3a26 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/codes_lad23.csv @@ -0,0 +1,362 @@ +code +E06000001 +E06000002 +E06000003 +E06000004 +E06000005 +E06000006 +E06000007 +E06000008 +E06000009 +E06000010 +E06000011 +E06000012 +E06000013 +E06000014 +E06000015 +E06000016 +E06000017 +E06000018 +E06000019 +E06000020 +E06000021 +E06000022 +E06000023 +E06000024 +E06000025 +E06000026 +E06000027 +E06000030 +E06000031 +E06000032 +E06000033 +E06000034 +E06000035 +E06000036 +E06000037 +E06000038 +E06000039 +E06000040 +E06000041 +E06000042 +E06000043 +E06000044 +E06000045 +E06000046 +E06000047 +E06000049 +E06000050 +E06000051 +E06000052 +E06000053 +E06000054 +E06000055 +E06000056 +E06000057 +E06000058 +E06000059 +E06000060 +E06000061 +E06000062 +E06000063 +E06000064 +E06000065 +E06000066 +E07000008 +E07000009 +E07000010 +E07000011 +E07000012 +E07000032 +E07000033 +E07000034 +E07000035 +E07000036 +E07000037 +E07000038 +E07000039 +E07000040 +E07000041 +E07000042 +E07000043 +E07000044 +E07000045 +E07000046 +E07000047 +E07000061 +E07000062 +E07000063 +E07000064 +E07000065 +E07000066 +E07000067 +E07000068 +E07000069 +E07000070 +E07000071 +E07000072 +E07000073 +E07000074 +E07000075 +E07000076 +E07000077 +E07000078 +E07000079 +E07000080 +E07000081 +E07000082 +E07000083 +E07000084 +E07000085 +E07000086 +E07000087 +E07000088 +E07000089 +E07000090 +E07000091 +E07000092 +E07000093 +E07000094 +E07000095 +E07000096 +E07000098 +E07000099 +E07000102 +E07000103 +E07000105 +E07000106 +E07000107 +E07000108 +E07000109 +E07000110 +E07000111 +E07000112 +E07000113 +E07000114 +E07000115 +E07000116 +E07000117 +E07000118 +E07000119 +E07000120 +E07000121 +E07000122 +E07000123 +E07000124 +E07000125 +E07000126 +E07000127 +E07000128 +E07000129 +E07000130 +E07000131 +E07000132 +E07000133 +E07000134 +E07000135 +E07000136 +E07000137 +E07000138 +E07000139 +E07000140 +E07000141 +E07000142 +E07000143 +E07000144 +E07000145 +E07000146 +E07000147 +E07000148 +E07000149 +E07000170 +E07000171 +E07000172 +E07000173 +E07000174 +E07000175 +E07000176 +E07000177 +E07000178 +E07000179 +E07000180 +E07000181 +E07000192 +E07000193 +E07000194 +E07000195 +E07000196 +E07000197 +E07000198 +E07000199 +E07000200 +E07000202 +E07000203 +E07000207 +E07000208 +E07000209 +E07000210 +E07000211 +E07000212 +E07000213 +E07000214 +E07000215 +E07000216 +E07000217 +E07000218 +E07000219 +E07000220 +E07000221 +E07000222 +E07000223 +E07000224 +E07000225 +E07000226 +E07000227 +E07000228 +E07000229 +E07000234 +E07000235 +E07000236 +E07000237 +E07000238 +E07000239 +E07000240 +E07000241 +E07000242 +E07000243 +E07000244 +E07000245 +E08000001 +E08000002 +E08000003 +E08000004 +E08000005 +E08000006 +E08000007 +E08000008 +E08000009 +E08000010 +E08000011 +E08000012 +E08000013 +E08000014 +E08000015 +E08000016 +E08000017 +E08000018 +E08000019 +E08000021 +E08000022 +E08000023 +E08000024 +E08000025 +E08000026 +E08000027 +E08000028 +E08000029 +E08000030 +E08000031 +E08000032 +E08000033 +E08000034 +E08000035 +E08000036 +E08000037 +E09000001 +E09000002 +E09000003 +E09000004 +E09000005 +E09000006 +E09000007 +E09000008 +E09000009 +E09000010 +E09000011 +E09000012 +E09000013 +E09000014 +E09000015 +E09000016 +E09000017 +E09000018 +E09000019 +E09000020 +E09000021 +E09000022 +E09000023 +E09000024 +E09000025 +E09000026 +E09000027 +E09000028 +E09000029 +E09000030 +E09000031 +E09000032 +E09000033 +N09000001 +N09000002 +N09000003 +N09000004 +N09000005 +N09000006 +N09000007 +N09000008 +N09000009 +N09000010 +N09000011 +S12000005 +S12000006 +S12000008 +S12000010 +S12000011 +S12000013 +S12000014 +S12000017 +S12000018 +S12000019 +S12000020 +S12000021 +S12000023 +S12000026 +S12000027 +S12000028 +S12000029 +S12000030 +S12000033 +S12000034 +S12000035 +S12000036 +S12000038 +S12000039 +S12000040 +S12000041 +S12000042 +S12000045 +S12000047 +S12000048 +S12000049 +S12000050 +W06000001 +W06000002 +W06000003 +W06000004 +W06000005 +W06000006 +W06000008 +W06000009 +W06000010 +W06000011 +W06000012 +W06000013 +W06000014 +W06000015 +W06000016 +W06000018 +W06000019 +W06000020 +W06000021 +W06000022 +W06000023 +W06000024 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv deleted file mode 100644 index cc333e371..000000000 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad22.csv +++ /dev/null @@ -1,375 +0,0 @@ -code,x,y -E06000001,8,19 -E06000002,9,18 -E06000003,9,19 -E06000004,8,18 -E06000005,7,18 -E06000006,1,11 -E06000007,2,11 -E06000008,4,15 -E06000009,2,15 -E06000010,10,15 -E06000011,11,16 -E06000012,11,14 -E06000013,10,14 -E06000014,9,17 -E06000015,6,11 -E06000016,8,8 -E06000017,10,9 -E06000018,8,10 -E06000019,0,8 -E06000020,2,9 -E06000021,3,10 -E06000022,1,3 -E06000023,0,3 -E06000024,0,2 -E06000025,1,4 -E06000026,-4,-2 -E06000027,-3,-2 -E06000030,2,4 -E06000031,11,9 -E06000032,10,7 -E06000033,16,6 -E06000034,15,4 -E06000035,15,1 -E06000036,4,2 -E06000037,2,2 -E06000038,2,3 -E06000039,6,4 -E06000040,4,3 -E06000041,3,3 -E06000042,6,5 -E06000043,9,-2 -E06000044,4,-1 -E06000045,2,0 -E06000046,1,-2 -E06000047,6,18 -E06000049,4,11 -E06000050,3,11 -E06000051,1,9 -E06000052,-5,-2 -E06000053,-7,-3 -E06000054,1,2 -E06000055,9,7 -E06000056,9,6 -E06000057,5,20 -E06000058,0,0 -E06000059,-1,0 -E06000060,5,5 -E06000061,9,9 -E06000062,7,6 -E07000008,12,8 -E07000009,12,9 -E07000010,13,10 -E07000011,10,8 -E07000012,11,8 -E07000026,4,18 -E07000027,2,17 -E07000028,4,19 -E07000029,3,18 -E07000030,5,18 -E07000031,4,17 -E07000032,7,11 -E07000033,10,12 -E07000034,9,12 -E07000035,7,12 -E07000036,7,9 -E07000037,7,13 -E07000038,8,12 -E07000039,6,10 -E07000040,-2,-1 -E07000041,-3,-1 -E07000042,-2,0 -E07000043,-3,1 -E07000044,-4,-3 -E07000045,-2,-2 -E07000046,-4,-1 -E07000047,-3,0 -E07000061,10,-2 -E07000062,13,-2 -E07000063,10,-1 -E07000064,12,-2 -E07000065,11,-2 -E07000066,14,5 -E07000067,14,7 -E07000068,13,5 -E07000069,15,5 -E07000070,14,6 -E07000071,15,8 -E07000072,12,5 -E07000073,13,6 -E07000074,15,7 -E07000075,15,6 -E07000076,16,8 -E07000077,13,7 -E07000078,1,5 -E07000079,2,5 -E07000080,-1,6 -E07000081,0,6 -E07000082,0,5 -E07000083,1,6 -E07000084,2,1 -E07000085,4,0 -E07000086,3,0 -E07000087,2,-1 -E07000088,3,-1 -E07000089,3,2 -E07000090,5,0 -E07000091,1,0 -E07000092,4,1 -E07000093,1,1 -E07000094,3,1 -E07000095,12,6 -E07000096,8,6 -E07000098,9,5 -E07000099,11,7 -E07000102,7,5 -E07000103,8,5 -E07000105,12,-1 -E07000106,15,0 -E07000107,13,1 -E07000108,14,-1 -E07000109,14,1 -E07000110,14,0 -E07000111,12,0 -E07000112,13,-1 -E07000113,16,0 -E07000114,15,-1 -E07000115,13,0 -E07000116,11,-1 -E07000117,6,15 -E07000118,3,14 -E07000119,4,16 -E07000120,5,15 -E07000121,3,17 -E07000122,6,16 -E07000123,5,16 -E07000124,5,17 -E07000125,6,14 -E07000126,3,15 -E07000127,2,13 -E07000128,3,16 -E07000129,7,7 -E07000130,8,9 -E07000131,8,7 -E07000132,7,8 -E07000133,11,10 -E07000134,6,9 -E07000135,9,8 -E07000136,12,12 -E07000137,12,13 -E07000138,11,12 -E07000139,11,11 -E07000140,12,11 -E07000141,12,10 -E07000142,11,13 -E07000143,14,10 -E07000144,15,12 -E07000145,15,11 -E07000146,13,11 -E07000147,14,12 -E07000148,14,11 -E07000149,15,10 -E07000163,6,17 -E07000164,10,18 -E07000165,8,17 -E07000166,7,17 -E07000167,10,16 -E07000168,10,17 -E07000169,9,16 -E07000170,8,11 -E07000171,10,13 -E07000172,7,10 -E07000173,9,10 -E07000174,9,11 -E07000175,10,11 -E07000176,10,10 -E07000177,4,5 -E07000178,4,4 -E07000179,5,4 -E07000180,3,4 -E07000181,3,5 -E07000187,0,1 -E07000188,-1,2 -E07000189,-1,1 -E07000192,3,9 -E07000193,5,11 -E07000194,4,9 -E07000195,2,10 -E07000196,2,8 -E07000197,4,10 -E07000198,5,10 -E07000199,5,9 -E07000200,14,8 -E07000202,15,9 -E07000203,14,9 -E07000207,7,2 -E07000208,8,0 -E07000209,5,1 -E07000210,6,1 -E07000211,7,0 -E07000212,5,3 -E07000213,6,3 -E07000214,5,2 -E07000215,9,-1 -E07000216,6,0 -E07000217,6,2 -E07000218,6,8 -E07000219,6,7 -E07000220,6,6 -E07000221,3,6 -E07000222,4,6 -E07000223,8,-2 -E07000224,6,-2 -E07000225,5,-1 -E07000226,8,-1 -E07000227,6,-1 -E07000228,7,-1 -E07000229,7,-2 -E07000234,2,7 -E07000235,-1,7 -E07000236,4,7 -E07000237,0,7 -E07000238,2,6 -E07000239,1,8 -E07000240,10,6 -E07000241,11,6 -E07000242,13,8 -E07000243,12,7 -E07000244,16,10 -E07000245,13,9 -E07000246,-2,1 -E08000001,4,14 -E08000002,5,14 -E08000003,5,12 -E08000004,5,13 -E08000005,7,14 -E08000006,4,13 -E08000007,6,12 -E08000008,6,13 -E08000009,4,12 -E08000010,3,13 -E08000011,2,12 -E08000012,1,13 -E08000013,3,12 -E08000014,2,14 -E08000015,1,12 -E08000016,8,14 -E08000017,9,14 -E08000018,9,13 -E08000019,8,13 -E08000021,5,19 -E08000022,6,20 -E08000023,7,20 -E08000024,7,19 -E08000025,5,8 -E08000026,5,6 -E08000027,1,7 -E08000028,3,7 -E08000029,5,7 -E08000030,4,8 -E08000031,3,8 -E08000032,7,16 -E08000033,7,15 -E08000034,8,15 -E08000035,8,16 -E08000036,9,15 -E08000037,6,19 -E09000001,11,2 -E09000002,13,3 -E09000003,10,5 -E09000004,12,1 -E09000005,10,4 -E09000006,11,0 -E09000007,11,4 -E09000008,10,0 -E09000009,9,4 -E09000010,11,5 -E09000011,11,1 -E09000012,12,3 -E09000013,8,3 -E09000014,12,4 -E09000015,8,4 -E09000016,14,3 -E09000017,7,4 -E09000018,7,3 -E09000019,11,3 -E09000020,9,3 -E09000021,7,1 -E09000022,10,2 -E09000023,10,1 -E09000024,8,1 -E09000025,13,2 -E09000026,14,4 -E09000027,8,2 -E09000028,9,1 -E09000029,9,0 -E09000030,12,2 -E09000031,13,4 -E09000032,9,2 -E09000033,10,3 -N09000001,-4,16 -N09000002,-5,16 -N09000003,-4,17 -N09000004,-5,18 -N09000005,-6,17 -N09000006,-6,16 -N09000007,-5,15 -N09000008,-4,18 -N09000009,-5,17 -N09000010,-4,15 -N09000011,-3,16 -S12000005,2,24 -S12000006,4,20 -S12000008,3,20 -S12000010,5,22 -S12000011,2,20 -S12000013,-1,27 -S12000014,2,23 -S12000017,1,26 -S12000018,0,21 -S12000019,3,21 -S12000020,2,26 -S12000021,1,20 -S12000023,4,28 -S12000026,4,21 -S12000027,5,30 -S12000028,1,19 -S12000029,2,21 -S12000030,1,24 -S12000033,4,26 -S12000034,3,26 -S12000035,0,24 -S12000036,4,22 -S12000038,1,22 -S12000039,0,23 -S12000040,3,22 -S12000041,2,25 -S12000042,3,25 -S12000045,1,23 -S12000047,3,24 -S12000048,1,25 -S12000049,1,21 -S12000050,2,22 -W06000001,-2,12 -W06000002,-2,10 -W06000003,-1,10 -W06000004,0,10 -W06000005,0,11 -W06000006,1,10 -W06000008,-2,9 -W06000009,-5,6 -W06000010,-4,6 -W06000011,-4,5 -W06000012,-3,5 -W06000013,-3,6 -W06000014,-2,4 -W06000015,-2,5 -W06000016,-3,7 -W06000018,-2,6 -W06000019,0,9 -W06000020,-2,7 -W06000021,-1,8 -W06000022,-1,5 -W06000023,-1,9 -W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv deleted file mode 100644 index 591d36189..000000000 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/coordinates_lad23.csv +++ /dev/null @@ -1,362 +0,0 @@ -code,x,y -E06000001,8,19 -E06000002,9,18 -E06000003,9,19 -E06000004,8,18 -E06000005,7,18 -E06000006,1,11 -E06000007,2,11 -E06000008,4,15 -E06000009,2,15 -E06000010,10,15 -E06000011,11,16 -E06000012,11,14 -E06000013,10,14 -E06000014,9,17 -E06000015,6,11 -E06000016,8,8 -E06000017,10,9 -E06000018,8,10 -E06000019,0,8 -E06000020,2,9 -E06000021,3,10 -E06000022,1,3 -E06000023,0,3 -E06000024,0,2 -E06000025,1,4 -E06000026,-4,-2 -E06000027,-3,-2 -E06000030,2,4 -E06000031,11,9 -E06000032,10,7 -E06000033,16,6 -E06000034,15,4 -E06000035,15,1 -E06000036,4,2 -E06000037,2,2 -E06000038,2,3 -E06000039,6,4 -E06000040,4,3 -E06000041,3,3 -E06000042,6,5 -E06000043,9,-2 -E06000044,4,-1 -E06000045,2,0 -E06000046,1,-2 -E06000047,6,18 -E06000049,4,11 -E06000050,3,11 -E06000051,1,9 -E06000052,-5,-2 -E06000053,-7,-3 -E06000054,1,2 -E06000055,9,7 -E06000056,9,6 -E06000057,5,20 -E06000058,0,0 -E06000059,-1,0 -E06000060,5,5 -E06000061,9,9 -E06000062,7,6 -E06000063,4,18 -E06000064,4,17 -E06000065,8,17 -E06000066,0,1 -E07000008,12,8 -E07000009,12,9 -E07000010,13,10 -E07000011,10,8 -E07000012,11,8 -E07000032,7,11 -E07000033,10,12 -E07000034,9,12 -E07000035,7,12 -E07000036,7,9 -E07000037,7,13 -E07000038,8,12 -E07000039,6,10 -E07000040,-2,-1 -E07000041,-3,-1 -E07000042,-2,0 -E07000043,-3,1 -E07000044,-4,-3 -E07000045,-2,-2 -E07000046,-4,-1 -E07000047,-3,0 -E07000061,10,-2 -E07000062,13,-2 -E07000063,10,-1 -E07000064,12,-2 -E07000065,11,-2 -E07000066,14,5 -E07000067,14,7 -E07000068,13,5 -E07000069,15,5 -E07000070,14,6 -E07000071,15,8 -E07000072,12,5 -E07000073,13,6 -E07000074,15,7 -E07000075,15,6 -E07000076,16,8 -E07000077,13,7 -E07000078,1,5 -E07000079,2,5 -E07000080,-1,6 -E07000081,0,6 -E07000082,0,5 -E07000083,1,6 -E07000084,2,1 -E07000085,4,0 -E07000086,3,0 -E07000087,2,-1 -E07000088,3,-1 -E07000089,3,2 -E07000090,5,0 -E07000091,1,0 -E07000092,4,1 -E07000093,1,1 -E07000094,3,1 -E07000095,12,6 -E07000096,8,6 -E07000098,9,5 -E07000099,11,7 -E07000102,7,5 -E07000103,8,5 -E07000105,12,-1 -E07000106,15,0 -E07000107,13,1 -E07000108,14,-1 -E07000109,14,1 -E07000110,14,0 -E07000111,12,0 -E07000112,13,-1 -E07000113,16,0 -E07000114,15,-1 -E07000115,13,0 -E07000116,11,-1 -E07000117,6,15 -E07000118,3,14 -E07000119,4,16 -E07000120,5,15 -E07000121,3,17 -E07000122,6,16 -E07000123,5,16 -E07000124,5,17 -E07000125,6,14 -E07000126,3,15 -E07000127,2,13 -E07000128,3,16 -E07000129,7,7 -E07000130,8,9 -E07000131,8,7 -E07000132,7,8 -E07000133,11,10 -E07000134,6,9 -E07000135,9,8 -E07000136,12,12 -E07000137,12,13 -E07000138,11,12 -E07000139,11,11 -E07000140,12,11 -E07000141,12,10 -E07000142,11,13 -E07000143,14,10 -E07000144,15,12 -E07000145,15,11 -E07000146,13,11 -E07000147,14,12 -E07000148,14,11 -E07000149,15,10 -E07000170,8,11 -E07000171,10,13 -E07000172,7,10 -E07000173,9,10 -E07000174,9,11 -E07000175,10,11 -E07000176,10,10 -E07000177,4,5 -E07000178,4,4 -E07000179,5,4 -E07000180,3,4 -E07000181,3,5 -E07000192,3,9 -E07000193,5,11 -E07000194,4,9 -E07000195,2,10 -E07000196,2,8 -E07000197,4,10 -E07000198,5,10 -E07000199,5,9 -E07000200,14,8 -E07000202,15,9 -E07000203,14,9 -E07000207,7,2 -E07000208,8,0 -E07000209,5,1 -E07000210,6,1 -E07000211,7,0 -E07000212,5,3 -E07000213,6,3 -E07000214,5,2 -E07000215,9,-1 -E07000216,6,0 -E07000217,6,2 -E07000218,6,8 -E07000219,6,7 -E07000220,6,6 -E07000221,3,6 -E07000222,4,6 -E07000223,8,-2 -E07000224,6,-2 -E07000225,5,-1 -E07000226,8,-1 -E07000227,6,-1 -E07000228,7,-1 -E07000229,7,-2 -E07000234,2,7 -E07000235,-1,7 -E07000236,4,7 -E07000237,0,7 -E07000238,2,6 -E07000239,1,8 -E07000240,10,6 -E07000241,11,6 -E07000242,13,8 -E07000243,12,7 -E07000244,16,10 -E07000245,13,9 -E08000001,4,14 -E08000002,5,14 -E08000003,5,12 -E08000004,5,13 -E08000005,7,14 -E08000006,4,13 -E08000007,6,12 -E08000008,6,13 -E08000009,4,12 -E08000010,3,13 -E08000011,2,12 -E08000012,1,13 -E08000013,3,12 -E08000014,2,14 -E08000015,1,12 -E08000016,8,14 -E08000017,9,14 -E08000018,9,13 -E08000019,8,13 -E08000021,5,19 -E08000022,6,20 -E08000023,7,20 -E08000024,7,19 -E08000025,5,8 -E08000026,5,6 -E08000027,1,7 -E08000028,3,7 -E08000029,5,7 -E08000030,4,8 -E08000031,3,8 -E08000032,7,16 -E08000033,7,15 -E08000034,8,15 -E08000035,8,16 -E08000036,9,15 -E08000037,6,19 -E09000001,11,2 -E09000002,13,3 -E09000003,10,5 -E09000004,12,1 -E09000005,10,4 -E09000006,11,0 -E09000007,11,4 -E09000008,10,0 -E09000009,9,4 -E09000010,11,5 -E09000011,11,1 -E09000012,12,3 -E09000013,8,3 -E09000014,12,4 -E09000015,8,4 -E09000016,14,3 -E09000017,7,4 -E09000018,7,3 -E09000019,11,3 -E09000020,9,3 -E09000021,7,1 -E09000022,10,2 -E09000023,10,1 -E09000024,8,1 -E09000025,13,2 -E09000026,14,4 -E09000027,8,2 -E09000028,9,1 -E09000029,9,0 -E09000030,12,2 -E09000031,13,4 -E09000032,9,2 -E09000033,10,3 -N09000001,-4,16 -N09000002,-5,16 -N09000003,-4,17 -N09000004,-5,18 -N09000005,-6,17 -N09000006,-6,16 -N09000007,-5,15 -N09000008,-4,18 -N09000009,-5,17 -N09000010,-4,15 -N09000011,-3,16 -S12000005,2,24 -S12000006,4,20 -S12000008,3,20 -S12000010,5,22 -S12000011,2,20 -S12000013,-1,27 -S12000014,2,23 -S12000017,1,26 -S12000018,0,21 -S12000019,3,21 -S12000020,2,26 -S12000021,1,20 -S12000023,4,28 -S12000026,4,21 -S12000027,5,30 -S12000028,1,19 -S12000029,2,21 -S12000030,1,24 -S12000033,4,26 -S12000034,3,26 -S12000035,0,24 -S12000036,4,22 -S12000038,1,22 -S12000039,0,23 -S12000040,3,22 -S12000041,2,25 -S12000042,3,25 -S12000045,1,23 -S12000047,3,24 -S12000048,1,25 -S12000049,1,21 -S12000050,2,22 -W06000001,-2,12 -W06000002,-2,10 -W06000003,-1,10 -W06000004,0,10 -W06000005,0,11 -W06000006,1,10 -W06000008,-2,9 -W06000009,-5,6 -W06000010,-4,6 -W06000011,-4,5 -W06000012,-3,5 -W06000013,-3,6 -W06000014,-2,4 -W06000015,-2,5 -W06000016,-3,7 -W06000018,-2,6 -W06000019,0,9 -W06000020,-2,7 -W06000021,-1,8 -W06000022,-1,5 -W06000023,-1,9 -W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py index 18194145d..70766120a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py @@ -76,47 +76,24 @@ class LocalAuthorityBoundaryDefinition: ) -@dataclass(frozen=True, slots=True) -class LocalAuthorityCoordinate: - """Integer rendering coordinate for one local authority.""" - - x: int - y: int - - -@dataclass(frozen=True, slots=True) -class LocalAuthorityDisplayMetadata: - """Display-only metadata attached after authority impacts are calculated.""" - - name: str - x: int - y: int - - @dataclass(frozen=True, slots=True) class UKLocalAuthorityResources: - """Validated display names and coordinates for supported boundary versions.""" + """Validated display names and each supported boundary version's codes. + + ``names`` maps every supported authority code to its display name, and + ``codes`` lists the authority codes of each boundary version. + """ names: Mapping[str, str] - coordinates: Mapping[ - UKLocalAuthorityBoundaryVersion, - Mapping[str, LocalAuthorityCoordinate], - ] + codes: Mapping[UKLocalAuthorityBoundaryVersion, frozenset[str]] - def metadata_for( + def names_for( self, boundary_version: UKLocalAuthorityBoundaryVersion, - ) -> Mapping[str, LocalAuthorityDisplayMetadata]: - """Combine common names with one boundary version's rendering coordinates.""" + ) -> Mapping[str, str]: + """Return the display name of each authority in one boundary version.""" return MappingProxyType( - { - code: LocalAuthorityDisplayMetadata( - name=self.names[code], - x=coordinate.x, - y=coordinate.y, - ) - for code, coordinate in self.coordinates[boundary_version].items() - } + {code: self.names[code] for code in sorted(self.codes[boundary_version])} ) @@ -167,55 +144,39 @@ def _load_names() -> Mapping[str, str]: return MappingProxyType(names) -def _load_coordinates(filename: str) -> Mapping[str, LocalAuthorityCoordinate]: - coordinates: dict[str, LocalAuthorityCoordinate] = {} - for row in _rows(filename, ("code", "x", "y")): +def _load_codes(filename: str) -> frozenset[str]: + codes: set[str] = set() + for row in _rows(filename, ("code",)): code = row["code"] _require_code(code, filename) - if code in coordinates: + if code in codes: msg = f"UK local-authority resource {filename!r} repeats {code!r}" raise ValueError(msg) - try: - coordinate = LocalAuthorityCoordinate(x=int(row["x"]), y=int(row["y"])) - except ValueError as error: - msg = ( - f"UK local-authority resource {filename!r} has invalid coordinates " - f"for {code!r}" - ) - raise ValueError(msg) from error - coordinates[code] = coordinate - return MappingProxyType(coordinates) + codes.add(code) + return frozenset(codes) @lru_cache(maxsize=1) def load_uk_local_authority_resources() -> UKLocalAuthorityResources: """Load and validate every packaged local-authority resource.""" names = _load_names() - lad22 = _load_coordinates("coordinates_lad22.csv") - lad23 = _load_coordinates("coordinates_lad23.csv") - supported_codes = lad22.keys() | lad23.keys() - if names.keys() != supported_codes: + lad22 = _load_codes("codes_lad22.csv") + lad23 = _load_codes("codes_lad23.csv") + if names.keys() != lad22 | lad23: msg = ( "UK local-authority names must exactly cover the supported " "boundary versions" ) raise ValueError(msg) - if lad22.keys() - lad23.keys() != _LAD22_ONLY_CODES: - msg = "LAD22 distinguishing codes do not match its coordinate file" + if lad22 - lad23 != _LAD22_ONLY_CODES: + msg = "LAD22 distinguishing codes do not match its code file" raise ValueError(msg) - if lad23.keys() - lad22.keys() != _LAD23_ONLY_CODES: - msg = "LAD23 distinguishing codes do not match its coordinate file" + if lad23 - lad22 != _LAD23_ONLY_CODES: + msg = "LAD23 distinguishing codes do not match its code file" raise ValueError(msg) - for code in lad22.keys() & lad23.keys(): - if lad22[code] != lad23[code]: - msg = ( - "UK local-authority coordinate differs between boundary versions " - f"for {code!r}" - ) - raise ValueError(msg) return UKLocalAuthorityResources( names=names, - coordinates=MappingProxyType( + codes=MappingProxyType( { UKLocalAuthorityBoundaryVersion.LAD22: lad22, UKLocalAuthorityBoundaryVersion.LAD23: lad23, diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_fakes.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_fakes.py index ad4bb8d9b..5e80312e6 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_fakes.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_fakes.py @@ -198,8 +198,6 @@ def constituency_impact_record() -> ConstituencyImpactRecord: return ConstituencyImpactRecord( constituency_code="E14000530", constituency_name="Birmingham, Ladywood", - x=34, - y=-42, average_household_income_change=10.0, relative_household_income_change=0.01, population=1000.0, @@ -210,8 +208,6 @@ def local_authority_impact_record() -> LocalAuthorityImpactRecord: return LocalAuthorityImpactRecord( local_authority_code="E06000001", local_authority_name="Hartlepool", - x=63, - y=-19, average_household_income_change=12.0, relative_household_income_change=0.02, population=900.0, diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index 84636b533..05b27f12b 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -22,8 +22,6 @@ def _complete_constituency_record() -> dict[str, object]: return { "constituency_code": "E14001063", "constituency_name": "Aldershot", - "x": 56, - "y": -40, "average_household_income_change": -150.95, "relative_household_income_change": -0.0026, "population": 40_530.65, @@ -85,8 +83,8 @@ def test_complete_uk_geography_output_accepts_lookup_metadata() -> None: record = result.root[0].model_dump(mode="python") assert record["constituency_name"] == "Aldershot" - assert record["x"] == 56 - assert record["y"] == -40 + assert "x" not in record + assert "y" not in record def test_complete_uk_geography_output_accepts_an_empty_list() -> None: @@ -114,8 +112,6 @@ def test_complete_uk_geography_output_rejects_a_missing_list() -> None: [ ("constituency_name", "E14001063", "missing a lookup name"), ("constituency_name", "", "missing a lookup name"), - ("x", None, "missing lookup coordinates"), - ("y", None, "missing lookup coordinates"), ], ) def test_complete_uk_geography_output_rejects_fallback_metadata( @@ -286,6 +282,10 @@ def test_uk_constituency_output_lists_only_published_seats( for record in records: change = published[record["constituency_code"]] assert record["constituency_name"].startswith("Seat ") + # policyengine.py's lookup still has x and y columns; sim-api no + # longer sends an area's map position. + assert "x" not in record + assert "y" not in record assert record["average_household_income_change"] == change.average assert record["relative_household_income_change"] == change.relative assert record["disclosure_tier"] == change.tier @@ -322,14 +322,32 @@ def test_uk_local_authority_output_uses_detected_boundary_metadata( "Cumberland", "Westmorland and Furness", } - assert isinstance(record["x"], int) - assert isinstance(record["y"], int) + assert "x" not in record + assert "y" not in record assert record["average_household_income_change"] < 0 assert record["population"] == ( uk_disclosure.level("local_authority")[code].population ) +def test_uk_area_outputs_serialize_without_x_or_y( + uk_disclosure, constituency_lookup +) -> None: + constituencies = simulation_output_geographic.build_uk_constituency_impact( + "uk", uk_disclosure + ) + authorities = simulation_output_geographic.build_uk_local_authority_impact( + "uk", uk_disclosure, uk_local_authority_metadata=LAD23 + ) + + assert constituencies is not None + assert authorities is not None + for output in (constituencies, authorities): + records = output.model_dump(mode="json") + assert records + assert all("x" not in record and "y" not in record for record in records) + + def test_uk_local_authority_output_rejects_code_outside_boundary_version( uk_disclosure, ) -> None: diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index 671aa4d12..e511ee05e 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -213,8 +213,8 @@ def test_v2_static_runtime_file_smoke_uses_exact_uk_worker_image(monkeypatch) -> ) assert smoke.smoke_stage12_static_runtime_files() == { "names": 378, - "lad22_coordinates": 374, - "lad23_coordinates": 361, + "lad22_codes": 374, + "lad23_codes": 361, "installed_dataset_boundary_version": "lad22", } diff --git a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py index 20552792b..853a3f6da 100644 --- a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py @@ -2,13 +2,19 @@ from __future__ import annotations +import hashlib +import re +import shutil +from importlib.resources import files from types import SimpleNamespace +from typing import TYPE_CHECKING import pandas as pd import pytest from policyengine_simulation_contract.uk_geography import ( UKLocalAuthorityBoundaryVersion, ) +from policyengine_simulation_executor import uk_local_authority_metadata from policyengine_simulation_executor.uk_local_authority_metadata import ( detect_uk_area_dataset_metadata, detect_uk_local_authority_boundary_version, @@ -22,6 +28,10 @@ uk_hdf_household_has_local_authority_codes, ) +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from pathlib import Path + LAD22_ONLY_CODES = { "E07000026", "E07000027", @@ -49,42 +59,140 @@ } +# SHA-256 of each boundary version's sorted codes, one per line, as listed by +# coordinates_lad22.csv and coordinates_lad23.csv before the executor stopped +# shipping local-authority map positions. The code lists that replaced those +# files must keep exactly these codes. +RETIRED_COORDINATE_FILE_CODE_DIGESTS = { + UKLocalAuthorityBoundaryVersion.LAD22: ( + "36b51d2d163e3e094eebd838daf3417a416ded29cd4d3b54af4901ae9c58e37f" + ), + UKLocalAuthorityBoundaryVersion.LAD23: ( + "9a313a86a8b4542d4020cc50d613fcce8d09dd806253f1716ff3e126a8ed1ab2" + ), +} +PACKAGED_RESOURCE_DIRECTORY = files("policyengine_simulation_executor").joinpath( + "static_runtime_files", "uk_local_authorities" +) + + def test_packaged_resources_cover_the_exact_supported_boundary_versions() -> None: resources = load_uk_local_authority_resources() - lad22 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - lad23 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] + lad22 = resources.codes[UKLocalAuthorityBoundaryVersion.LAD22] + lad23 = resources.codes[UKLocalAuthorityBoundaryVersion.LAD23] assert len(resources.names) == 378 assert len(lad22) == 374 assert len(lad23) == 361 - assert len(lad22.keys() & lad23.keys()) == 357 - assert lad22.keys() - lad23.keys() == LAD22_ONLY_CODES - assert lad23.keys() - lad22.keys() == LAD23_ONLY_CODES - assert resources.names.keys() == lad22.keys() | lad23.keys() + assert len(lad22 & lad23) == 357 + assert lad22 - lad23 == LAD22_ONLY_CODES + assert lad23 - lad22 == LAD23_ONLY_CODES + assert resources.names.keys() == lad22 | lad23 + + +@pytest.mark.parametrize( + ("boundary_version", "digest"), RETIRED_COORDINATE_FILE_CODE_DIGESTS.items() +) +def test_each_boundary_version_keeps_the_codes_its_coordinate_file_listed( + boundary_version: UKLocalAuthorityBoundaryVersion, digest: str +) -> None: + codes = load_uk_local_authority_resources().codes[boundary_version] + listed = "\n".join(sorted(codes)).encode() + assert hashlib.sha256(listed).hexdigest() == digest -def test_shared_authorities_have_identical_coordinates() -> None: + +def test_the_executor_ships_names_and_code_lists_without_map_positions() -> None: + packaged = sorted(entry.name for entry in PACKAGED_RESOURCE_DIRECTORY.iterdir()) + + assert packaged == ["codes_lad22.csv", "codes_lad23.csv", "names.csv"] + + +def test_names_for_a_boundary_version_cover_exactly_its_codes() -> None: resources = load_uk_local_authority_resources() - lad22 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD22] - lad23 = resources.coordinates[UKLocalAuthorityBoundaryVersion.LAD23] - for code in lad22.keys() & lad23.keys(): - assert lad22[code] == lad23[code] + for boundary_version in UKLocalAuthorityBoundaryVersion: + names = resources.names_for(boundary_version) + assert names.keys() == resources.codes[boundary_version] + lad23_names = resources.names_for(UKLocalAuthorityBoundaryVersion.LAD23) + assert lad23_names["E06000063"] == "Cumberland" + assert "E06000063" not in resources.names_for(UKLocalAuthorityBoundaryVersion.LAD22) + + +@pytest.fixture +def editable_resources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Path]: + """Load the packaged resources from copies a test may change.""" + for entry in PACKAGED_RESOURCE_DIRECTORY.iterdir(): + with entry.open("rb") as source, (tmp_path / entry.name).open("wb") as copy: + shutil.copyfileobj(source, copy) + monkeypatch.setattr( + uk_local_authority_metadata, "_resource_path", lambda name: tmp_path / name + ) + load_uk_local_authority_resources.cache_clear() + yield tmp_path + load_uk_local_authority_resources.cache_clear() + +def _rewrite_lines(path: Path, change: Callable[[list[str]], list[str]]) -> None: + lines = path.read_text(encoding="utf-8").splitlines() + path.write_text("\n".join(change(lines)) + "\n", encoding="utf-8") -def test_resource_lookup_combines_names_with_boundary_coordinates() -> None: + +def test_resources_load_from_unchanged_copies(editable_resources: Path) -> None: resources = load_uk_local_authority_resources() - cumberland = resources.metadata_for(UKLocalAuthorityBoundaryVersion.LAD23)[ - "E06000063" - ] - assert cumberland.name == "Cumberland" - assert isinstance(cumberland.x, int) - assert isinstance(cumberland.y, int) - assert "E06000063" not in resources.metadata_for( - UKLocalAuthorityBoundaryVersion.LAD22 + assert len(resources.codes[UKLocalAuthorityBoundaryVersion.LAD22]) == 374 + + +def test_resources_refuse_names_that_miss_a_supported_code( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "names.csv", + lambda lines: [line for line in lines if not line.startswith("E06000063,")], + ) + + with pytest.raises(ValueError, match="must exactly cover"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_code_list_with_other_distinguishing_codes( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "codes_lad23.csv", + # E07000026 is one of the codes only LAD22 has. + lambda lines: [lines[0], *sorted([*lines[1:], "E07000026"])], ) + with pytest.raises(ValueError, match="LAD22 distinguishing codes"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_repeated_code(editable_resources: Path) -> None: + _rewrite_lines( + editable_resources / "codes_lad22.csv", lambda lines: [*lines, lines[1]] + ) + + with pytest.raises(ValueError, match="repeats 'E06000001'"): + load_uk_local_authority_resources() + + +def test_resources_refuse_a_code_list_that_carries_map_positions( + editable_resources: Path, +) -> None: + _rewrite_lines( + editable_resources / "codes_lad22.csv", + lambda lines: ["code,x,y", *(f"{code},0,0" for code in lines[1:])], + ) + + with pytest.raises( + ValueError, match=re.escape("'codes_lad22.csv' has invalid columns") + ): + load_uk_local_authority_resources() + def test_detector_identifies_lad22_from_a_predecessor_code() -> None: metadata = detect_uk_local_authority_boundary_version(