Remove x and y from the contract's UK area records

changelog.d/923.changed.md

@@ -0,0 +1 @@
+UK constituency and local-authority records no longer have `x` or `y` fields in the contract or in the gateway's OpenAPI document. A record that carries either, even as `null`, is now refused. The contract's pydantic requirement returns to 2.0 or later, since nothing uses field-level `exclude_if` any more.

libs/policyengine-simulation-contract/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 license = {file = "../../LICENSE"}
 requires-python = ">=3.13"
 dependencies = [
-    "pydantic>=2.12",
+    "pydantic>=2.0",
     "packaging>=24",
     # gateway_models embeds the TelemetryEnvelope from the observability lib.
     "policyengine-simulation-observability",

libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py

@@ -4,7 +4,7 @@
 
 from typing import Generic, Literal, TypeVar
 
-from pydantic import BaseModel, ConfigDict, Field, RootModel
+from pydantic import BaseModel, ConfigDict, RootModel
 
 from policyengine_simulation_contract.spm import SPMComparisonProvenance
 
@@ -240,10 +240,6 @@ class CliffImpactOutput(MacroOutputModel):
 UKGeographyDisclosureTier = Literal["reported", "approximate"]
 
 
-def _is_none(value: object) -> bool:
-    return value is None
-
-
 class UKAreaDisclosureFields(MacroOutputModel):
     """Disclosure tier and 90% intervals of a published UK area's income changes.
 
@@ -272,16 +268,10 @@ class ConstituencyImpactRecord(UKAreaDisclosureFields):
     ``relative_household_income_change`` the relative change in the area's total
     household net income, both rounded to the order of their standard error.
     ``population`` is the weighted number of households.
-
-    ``x`` and ``y`` are the area's integer map position, which is being
-    removed: a record validates with or without them, and one without them
-    leaves both out of its JSON rather than writing null.
     """
 
     constituency_code: str
     constituency_name: str
-    x: int | None = Field(default=None, exclude_if=_is_none)
-    y: int | None = Field(default=None, exclude_if=_is_none)
     average_household_income_change: float
     relative_household_income_change: float
     population: float
@@ -295,8 +285,6 @@ class LocalAuthorityImpactRecord(UKAreaDisclosureFields):
 
     local_authority_code: str
     local_authority_name: str
-    x: int | None = Field(default=None, exclude_if=_is_none)
-    y: int | None = Field(default=None, exclude_if=_is_none)
     average_household_income_change: float
     relative_household_income_change: float
     population: float

libs/policyengine-simulation-contract/tests/test_local_area_breakdown.py

@@ -23,8 +23,6 @@
 CONSTITUENCY = {
     "constituency_code": "E14001063",
     "constituency_name": "Aldershot",
-    "x": 56,
-    "y": -40,
     "average_household_income_change": -150.0,
     "relative_household_income_change": -0.0026,
     "population": 40_530.0,
@@ -37,8 +35,6 @@
 LOCAL_AUTHORITY = {
     "local_authority_code": "E06000063",
     "local_authority_name": "Cumberland",
-    "x": 4,
-    "y": 18,
     "average_household_income_change": -120.0,
     "relative_household_income_change": -0.002,
     "population": 120_000.0,
@@ -203,43 +199,41 @@ def test_each_area_level_reads_its_records_as_its_own_type():
     ]
 
 
-def _without_map_position(record: dict[str, object]) -> dict[str, object]:
-    return {key: value for key, value in record.items() if key not in {"x", "y"}}
-
-
-# Records from an executor that no longer sends each area's map position.
-BREAKDOWN_WITHOUT_MAP_POSITIONS = {
-    **BREAKDOWN,
-    "constituency_impact": [_without_map_position(CONSTITUENCY)],
-    "local_authority_impact": [_without_map_position(LOCAL_AUTHORITY)],
-}
-
-
-def test_a_breakdown_without_map_positions_round_trips_without_them():
-    output = LocalAreaBreakdownOutput.model_validate(BREAKDOWN_WITHOUT_MAP_POSITIONS)
-
-    assert output.model_dump(mode="json") == BREAKDOWN_WITHOUT_MAP_POSITIONS
-
-
-def test_a_job_status_reads_a_breakdown_without_map_positions():
-    response = JobStatusResponse(
-        status="complete", result=BREAKDOWN_WITHOUT_MAP_POSITIONS
-    )
-
-    assert isinstance(response.result, LocalAreaBreakdownOutput)
-    assert response.model_dump(mode="json")["result"] == (
-        BREAKDOWN_WITHOUT_MAP_POSITIONS
-    )
-
-
-def test_a_national_result_reads_area_lists_without_map_positions():
+def test_a_national_result_reads_its_area_lists():
     national = {
         **NATIONAL,
-        "constituency_impact": [_without_map_position(CONSTITUENCY)],
-        "local_authority_impact": [_without_map_position(LOCAL_AUTHORITY)],
+        "constituency_impact": [CONSTITUENCY],
+        "local_authority_impact": [LOCAL_AUTHORITY],
     }
 
     response = JobStatusResponse(status="complete", result=national)
 
     assert isinstance(response.result, SingleYearMacroOutput)
     assert response.model_dump(mode="json")["result"] == national
+
+
+# The contract has no x or y (each area's map position), so a result whose
+# records carry either is refused.
+@pytest.mark.parametrize(
+    "result",
+    [
+        pytest.param(
+            {**BREAKDOWN, "constituency_impact": [{**CONSTITUENCY, "x": 56, "y": -40}]},
+            id="breakdown-constituency",
+        ),
+        pytest.param(
+            {
+                **BREAKDOWN,
+                "local_authority_impact": [{**LOCAL_AUTHORITY, "x": 4, "y": 18}],
+            },
+            id="breakdown-local-authority",
+        ),
+        pytest.param(
+            {**NATIONAL, "constituency_impact": [{**CONSTITUENCY, "x": 56, "y": -40}]},
+            id="national-constituency",
+        ),
+    ],
+)
+def test_a_job_status_refuses_area_records_with_x_and_y(result):
+    with pytest.raises(ValidationError, match="Extra inputs are not permitted"):
+        JobStatusResponse(status="complete", result=result)

libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py

@@ -12,11 +12,9 @@
 )
 from pydantic import ValidationError
 
-STORED_BEFORE_DISCLOSURE = {
+RECORD_WITHOUT_DISCLOSURE = {
     "constituency_code": "E14001063",
     "constituency_name": "Aldershot",
-    "x": 56,
-    "y": -40,
     "average_household_income_change": -150.95,
     "relative_household_income_change": -0.0026,
     "population": 40_530.65,
@@ -30,8 +28,8 @@
 }
 
 
-def test_records_stored_before_disclosure_control_still_validate() -> None:
-    record = ConstituencyImpactRecord.model_validate(STORED_BEFORE_DISCLOSURE)
+def test_records_without_disclosure_fields_still_validate() -> None:
+    record = ConstituencyImpactRecord.model_validate(RECORD_WITHOUT_DISCLOSURE)
 
     assert record.disclosure_tier is None
     assert record.average_household_income_change_lower is None
@@ -43,8 +41,6 @@ def test_records_carry_the_disclosure_tier_and_intervals() -> None:
         {
             "local_authority_code": "E06000063",
             "local_authority_name": "Cumberland",
-            "x": 4,
-            "y": 18,
             "average_household_income_change": -150.0,
             "relative_household_income_change": -0.0026,
             "population": 120_000.0,
@@ -60,12 +56,12 @@ def test_a_withheld_tier_is_not_a_record() -> None:
     # Withheld areas are left out of the list, so no record holds that tier.
     with pytest.raises(ValidationError):
         ConstituencyImpactRecord.model_validate(
-            {**STORED_BEFORE_DISCLOSURE, "disclosure_tier": "withheld"}
+            {**RECORD_WITHOUT_DISCLOSURE, "disclosure_tier": "withheld"}
         )
 
 
 def test_published_values_stay_required() -> None:
-    record = {**STORED_BEFORE_DISCLOSURE, **DISCLOSURE_FIELDS}
+    record = {**RECORD_WITHOUT_DISCLOSURE, **DISCLOSURE_FIELDS}
     record["average_household_income_change"] = None
 
     with pytest.raises(ValidationError):
@@ -76,17 +72,9 @@ def test_an_empty_area_list_is_valid() -> None:
     assert GeographicImpactOutput.model_validate([]).root == []
 
 
-# x and y, each area's integer map position, are being removed. The executor
-# sends them until a later change stops it, so a record validates with or
-# without them.
-CONSTITUENCY_WITHOUT_MAP_POSITION = {
-    **{
-        key: value
-        for key, value in STORED_BEFORE_DISCLOSURE.items()
-        if key not in {"x", "y"}
-    },
-    **DISCLOSURE_FIELDS,
-}
+# The records have no x or y (an area's integer map position), and a record
+# that carries either is refused.
+CONSTITUENCY_WITHOUT_MAP_POSITION = {**RECORD_WITHOUT_DISCLOSURE, **DISCLOSURE_FIELDS}
 LOCAL_AUTHORITY_WITHOUT_MAP_POSITION = {
     "local_authority_code": "E06000063",
     "local_authority_name": "Cumberland",
@@ -108,41 +96,35 @@ def test_an_empty_area_list_is_valid() -> None:
 
 
 @pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION)
-def test_records_without_a_map_position_validate_and_dump_without_one(
+def test_records_validate_and_dump_without_x_or_y(
     record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord],
     payload: dict[str, object],
 ) -> None:
     record = record_type.model_validate(payload)
 
-    assert record.x is None
-    assert record.y is None
+    assert {"x", "y"}.isdisjoint(record_type.model_fields)
     assert record.model_dump(mode="python") == payload
     assert record.model_dump(mode="json") == payload
     assert json.loads(record.model_dump_json()) == payload
 
 
 @pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION)
-def test_records_with_a_map_position_keep_it(
-    record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord],
-    payload: dict[str, object],
-) -> None:
-    with_map_position = {**payload, "x": 56, "y": -40}
-
-    record = record_type.model_validate(with_map_position)
-
-    assert (record.x, record.y) == (56, -40)
-    assert record.model_dump(mode="json") == with_map_position
-
-
-@pytest.mark.parametrize(("record_type", "payload"), AREA_RECORDS_WITHOUT_MAP_POSITION)
-def test_a_null_map_position_is_left_out_of_the_dump(
+@pytest.mark.parametrize(
+    "map_position",
+    [
+        pytest.param({"x": 56, "y": -40}, id="both"),
+        pytest.param({"x": 56}, id="x-only"),
+        pytest.param({"y": -40}, id="y-only"),
+        pytest.param({"x": None, "y": None}, id="null"),
+    ],
+)
+def test_records_with_x_or_y_are_refused(
     record_type: type[ConstituencyImpactRecord | LocalAuthorityImpactRecord],
     payload: dict[str, object],
+    map_position: dict[str, int | None],
 ) -> None:
-    record = record_type.model_validate({**payload, "x": None, "y": None})
-
-    assert record.model_dump(mode="json") == payload
-    assert json.loads(record.model_dump_json()) == payload
+    with pytest.raises(ValidationError, match="Extra inputs are not permitted"):
+        record_type.model_validate({**payload, **map_position})
 
 
 def test_an_area_list_without_map_positions_dumps_without_them() -> None:

projects/policyengine-apis-integ/tests/simulation/test_calculate.py

@@ -4,6 +4,7 @@
 that economy-wide simulations complete successfully.
 """
 
+import inspect
 import json
 import time
 from collections.abc import Mapping
@@ -14,10 +15,12 @@
 import pytest
 from policyengine_api_simulation_client import AuthenticatedClient, Client
 from policyengine_api_simulation_client.models import (
+    ConstituencyImpactRecord,
     JobStage,
     JobStatusResponse,
     JobSubmitResponse,
     LocalAreaBreakdownOutput,
+    LocalAuthorityImpactRecord,
     SimulationRequest,
 )
 
@@ -363,10 +366,6 @@ def test_result_assertions_accept_generated_client_objects() -> None:
 }
 
 
-def _without_x_and_y(record: Mapping) -> dict:
-    return {key: value for key, value in record.items() if key not in {"x", "y"}}
-
-
 def test_breakdown_assertions_accept_generated_client_objects() -> None:
     status = JobStatusResponse.from_dict(
         {"status": "complete", "result": _BREAKDOWN_RESULT}
@@ -376,33 +375,11 @@ def test_breakdown_assertions_accept_generated_client_objects() -> None:
     assert_uk_local_area_breakdown(status.result)
 
 
-def test_generated_client_reads_area_records_without_x_and_y() -> None:
-    # Once the executor stops sending x and y, records arrive without them.
-    breakdown = {
-        **_BREAKDOWN_RESULT,
-        "constituency_impact": [_without_x_and_y(_BREAKDOWN_CONSTITUENCY)],
-        "local_authority_impact": [_without_x_and_y(_BREAKDOWN_LOCAL_AUTHORITY)],
-    }
-
-    status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown})
-
-    assert isinstance(status.result, LocalAreaBreakdownOutput)
-    assert status.result.to_dict() == breakdown
-
-
-def test_generated_client_reads_area_records_with_x_and_y() -> None:
-    # Breakdowns that started before the executor stopped sending x and y
-    # still carry them, until the contract removes the two fields.
-    breakdown = {
-        **_BREAKDOWN_RESULT,
-        "constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}],
-        "local_authority_impact": [{**_BREAKDOWN_LOCAL_AUTHORITY, "x": 4, "y": 18}],
-    }
-
-    status = JobStatusResponse.from_dict({"status": "complete", "result": breakdown})
-
-    assert isinstance(status.result, LocalAreaBreakdownOutput)
-    assert status.result.to_dict() == breakdown
+@pytest.mark.parametrize(
+    "record_model", [ConstituencyImpactRecord, LocalAuthorityImpactRecord]
+)
+def test_generated_client_area_records_have_no_x_or_y(record_model: type) -> None:
+    assert {"x", "y"}.isdisjoint(inspect.signature(record_model).parameters)
 
 
 @pytest.mark.parametrize(

projects/policyengine-simulation-gateway/fixtures/gateway_responses.py

@@ -13,8 +13,6 @@
         {
             "constituency_code": "E14001063",
             "constituency_name": "Aldershot",
-            "x": 56,
-            "y": -40,
             "average_household_income_change": -150.0,
             "relative_household_income_change": -0.0026,
             "population": 40530.0,