From 6988fc1b35b312052549b99b8edce348c18f7da0 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Tue, 27 Oct 2026 15:57:59 -0400 Subject: [PATCH 1/2] Record when each policyengine.py route was published and latest changed The routing state gains two optional fields: published_at (when each policyengine.py version's route was first published) and latest_since (when latest.policyengine last changed: a publish that advances it, a forced latest or a rollback). A date is recorded only when the publish is that event, so states from before this change get no wrong, later date for what they already route. The gateway's /versions returns both; the golden OpenAPI and the deploy smoke fixture are regenerated. Fixes #960 Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 1 + .../gateway_models.py | 17 +++ .../tests/test_gateway_models.py | 2 + .../tests/fixtures/versions_body.json | 5 + .../src/modal/routing_publisher/legacy.py | 4 + .../src/modal/routing_publisher/state.py | 42 +++++- .../version_registry/test_routing_dates.py | 131 ++++++++++++++++++ .../route_resolution.py | 18 +++ .../version_routes.py | 4 + .../tests/golden/openapi.json | 20 +++ .../tests/test_endpoints.py | 2 + .../tests/test_versions_fixture.py | 7 + 12 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index c044351cc..4a284862c 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,6 +1,7 @@ - bump: patch changes: added: + - Recorded in the routing state when each policyengine.py route was first published (published_at, by version) and when latest.policyengine last changed (latest_since, set by a publish that advances it, a forced latest or a rollback), and returned both from the gateway's /versions; a date is recorded only when the publish is that event, so states written before this change show no date for what they already route instead of a later, wrong one, and both fields are optional, so earlier states still read and validate - Added the rule sim.quality.python-file-length, checked by a new file-length quality guard with proof fixtures that its test writes to a temporary directory: a tracked Python file has at most 800 lines, or at most 1,500 if it is test code (a file under a tests or fixtures directory, or a conftest.py); the 14 files over their limit on main are listed in a baseline that may only shrink, so the rule starts ratcheted, with a downgrade section, authorized by the maintainer, whose tracking issue closes when the rule is enforced - Added breakdown "local_areas" to the simulation request for UK requests whose region is absent, uk or one of country/england, country/scotland, country/wales and country/northern_ireland, refused with 422 for other regions (including country/ni), other countries, include_cliffs=true, segmented=true or a region_group (include_cliffs=false stays accepted); the job runs one baseline and reform on the local-area dataset alone, with the region's filter, and returns a LocalAreaBreakdownOutput (kind local_areas, model_version, policyengine_version, dataset, data_version, and constituency_impact and local_authority_impact as lists of their own record types), which the job status result now accepts alongside SingleYearMacroOutput - Declared local_area_breakdown version 1 in the routing bundle manifest of apps whose policyengine.py bundle routes both UK area region types to one local-area release dataset (regional_release_kind local_area; an area_coded_national bundle, or one routing the two types to different datasets, does not count as routed), and made the gateway refuse breakdown with 400, before spawning, for any route whose manifest does not declare it, including legacy routes and routes pinned to an older version, since older apps ignore the field and would return a national result diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py index 4da0a6f9a..e883d69dd 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py @@ -505,6 +505,23 @@ class VersionsResponse(BaseModel): "wrapper version: read its identity at `bundles[latest]`." ), ) + published_at: dict[str, str] = Field( + default_factory=dict, + description=( + "When each exact PolicyEngine wrapper version's route was first " + "published, as an ISO 8601 UTC time. Republishing a version keeps " + "its date; versions published before the gateway recorded dates " + "have no entry." + ), + ) + latest_since: str | None = Field( + default=None, + description=( + "When `policyengine.latest` last changed (a publish that advanced " + "it, a forced latest or a rollback), as an ISO 8601 UTC time; null " + "until a publish records it." + ), + ) class HealthResponse(BaseModel): diff --git a/libs/policyengine-simulation-contract/tests/test_gateway_models.py b/libs/policyengine-simulation-contract/tests/test_gateway_models.py index 266a6dafd..3f33fb1e1 100644 --- a/libs/policyengine-simulation-contract/tests/test_gateway_models.py +++ b/libs/policyengine-simulation-contract/tests/test_gateway_models.py @@ -507,6 +507,8 @@ def test_versions_response_has_named_maps(self): "us": {"latest": "1.500.0"}, "uk": {"latest": "2.66.0"}, "bundles": {}, + "published_at": {}, + "latest_since": None, } def test_health_and_readiness_statuses_are_constrained(self): diff --git a/projects/policyengine-simulation-entry/tests/fixtures/versions_body.json b/projects/policyengine-simulation-entry/tests/fixtures/versions_body.json index 38ff7d458..739581329 100644 --- a/projects/policyengine-simulation-entry/tests/fixtures/versions_body.json +++ b/projects/policyengine-simulation-entry/tests/fixtures/versions_body.json @@ -37,11 +37,16 @@ } } }, + "latest_since": "2026-09-08T10:00:00Z", "policyengine": { "6.2.6": "policyengine-simulation-py6-2-6", "6.2.7": "policyengine-simulation-py6-2-7", "latest": "6.2.7" }, + "published_at": { + "6.2.6": "2026-09-01T10:00:00Z", + "6.2.7": "2026-09-08T10:00:00Z" + }, "spm_capabilities": {}, "uk": { "2.125.1": "policyengine-simulation-py6-2-6", diff --git a/projects/policyengine-simulation-executor/src/modal/routing_publisher/legacy.py b/projects/policyengine-simulation-executor/src/modal/routing_publisher/legacy.py index 5005855ed..6a593d6bf 100644 --- a/projects/policyengine-simulation-executor/src/modal/routing_publisher/legacy.py +++ b/projects/policyengine-simulation-executor/src/modal/routing_publisher/legacy.py @@ -179,5 +179,9 @@ def merge_legacy_seed_into_current( merged["latest"][kind] = current_latest merged["bundles"].update(current["bundles"]) + if "published_at" in current: + merged.setdefault("published_at", {}).update(current["published_at"]) + if "latest_since" in current: + merged["latest_since"] = current["latest_since"] validate_routing_state(merged) return merged diff --git a/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py b/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py index 6bba68870..bf905186c 100644 --- a/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py +++ b/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py @@ -3,7 +3,8 @@ from collections.abc import Iterable, Mapping from copy import deepcopy from dataclasses import dataclass -from typing import Any, TypedDict +from datetime import UTC, datetime +from typing import Any, NotRequired, TypedDict from packaging.version import InvalidVersion, Version from src.modal.routing_publisher.bundles import build_bundle_manifest_metadata @@ -52,6 +53,18 @@ class RoutingState(TypedDict): # its manifests are only known to be mappings until validate_routing_state # checks them against their routes. bundles: dict[str, Mapping[str, Any]] + # When each policyengine version's route was first published, as an ISO + # 8601 UTC time. Republishing a version keeps its date. States written + # before this field existed have no entry until their next publish. + published_at: NotRequired[dict[str, str]] + # When latest.policyengine last changed: a publish that advanced it, a + # forced latest or a rollback. + latest_since: NotRequired[str] + + +def utc_now() -> str: + """Return the current UTC time as the routing state records it.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") def is_newer_version(candidate: str, current: str | None) -> bool: @@ -125,6 +138,13 @@ def coerce_routing_state(value: object) -> RoutingState: if isinstance(key, str) and isinstance(value, dict) } + published_at = _routing_map(value.get("published_at")) + if published_at: + coerced["published_at"] = published_at + latest_since = value.get("latest_since") + if isinstance(latest_since, str) and latest_since: + coerced["latest_since"] = latest_since + return coerced @@ -229,12 +249,17 @@ def build_next_routing_state( current_state: object, deployed: DeployedAppVersions, force_latest: bool = False, + now: str | None = None, ) -> RoutingState: """Return ``current_state`` with routes and the bundle manifest for a new app added. ``latest`` advances only to a newer policyengine version, unless ``force_latest`` (an intentional rollback). Raises ValueError when the US or UK version differs from the installed bundle's model version. + + A publish that adds a policyengine version's route records + ``published_at`` for it, and a change of ``latest.policyengine`` records + ``latest_since``; both take ``now`` (default: the current UTC time). """ app_name = deployed.app_name policyengine_version = deployed.policyengine_version @@ -256,11 +281,18 @@ def build_next_routing_state( ) state = coerce_routing_state(current_state) + newly_routed = policyengine_version not in state["routes"]["policyengine"] state["generation"] = f"{policyengine_version}:{app_name}" state["routes"]["policyengine"][policyengine_version] = app_name state["routes"]["us"][us_version] = app_name state["routes"]["uk"][uk_version] = app_name state["bundles"][policyengine_version] = manifest + timestamp = now or utc_now() + # Dates are recorded only when this publish is the event: a route that did + # not exist, or a change of latest. A state from before dates were recorded + # has no date for what it already routes, rather than a later, wrong one. + if newly_routed: + state.setdefault("published_at", {})[policyengine_version] = timestamp previous_latest = state["latest"].get("policyengine") should_advance = ( @@ -272,6 +304,8 @@ def build_next_routing_state( "us": us_version, "uk": uk_version, } + if previous_latest != policyengine_version: + state["latest_since"] = timestamp validate_routing_state( state, @@ -285,6 +319,7 @@ def build_restored_routing_state( current_state: object, policyengine_version: str, expected_latest: str | None = None, + now: str | None = None, ) -> RoutingState: """Point ``latest`` back at a policyengine.py version that was already published. @@ -298,6 +333,9 @@ def build_restored_routing_state( ``latest.policyengine`` is still that version (the release being rolled back), so a rollback never overwrites a newer release published meanwhile. + A rollback that moves ``latest.policyengine`` records ``latest_since`` + (``now``, default: the current UTC time). + To undo a release completely (its routes as well as ``latest``), restore the routing plan its deployment recorded (:func:`restore_routing_plan`): which app served a model version before cannot be read from the state. @@ -335,6 +373,8 @@ def build_restored_routing_state( "us": restored_models["us"], "uk": restored_models["uk"], } + if current_latest != policyengine_version: + state["latest_since"] = now or utc_now() state["generation"] = f"rollback:{policyengine_version}:{app_name}" validate_routing_state( state, diff --git a/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py b/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py new file mode 100644 index 000000000..3a740e73e --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py @@ -0,0 +1,131 @@ +"""When each policyengine route was published and when latest last changed.""" + +from __future__ import annotations + +from registry_test_support import ( + install_bundle_manifests, + versioned_app_name, +) +from src.modal.routing_publisher import state as routing_state +from src.modal.utils import update_version_registry as registry + +FIRST = "2026-10-01T09:00:00Z" +SECOND = "2026-10-08T09:00:00Z" +THIRD = "2026-10-15T09:00:00Z" + +MODELS = { + "6.9.0": ("2.1.0", "2.120.0"), + "6.10.0": ("2.2.1", "2.132.6"), + "6.10.1": ("2.2.1", "2.133.0"), +} + + +def _publish(monkeypatch, state, version, now, *, force_latest=False): + install_bundle_manifests(monkeypatch, MODELS) + us_version, uk_version = MODELS[version] + return registry.build_next_routing_state( + current_state=state, + deployed=registry.DeployedAppVersions( + app_name=versioned_app_name(version), + policyengine_version=version, + us_version=us_version, + uk_version=uk_version, + ), + force_latest=force_latest, + now=now, + ) + + +def test_a_first_publish_records_both_dates(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + + assert state["published_at"] == {"6.10.0": FIRST} + assert state["latest_since"] == FIRST + + +def test_republishing_a_version_keeps_its_dates(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + + state = _publish(monkeypatch, state, "6.10.0", SECOND) + + assert state["published_at"] == {"6.10.0": FIRST} + assert state["latest_since"] == FIRST + + +def test_a_newer_version_moves_latest_since(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + + state = _publish(monkeypatch, state, "6.10.1", SECOND) + + assert state["published_at"] == {"6.10.0": FIRST, "6.10.1": SECOND} + assert state["latest_since"] == SECOND + + +def test_an_older_version_is_published_without_moving_latest(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + + state = _publish(monkeypatch, state, "6.9.0", SECOND) + + assert state["latest"]["policyengine"] == "6.10.0" + assert state["published_at"]["6.9.0"] == SECOND + assert state["latest_since"] == FIRST + + +def test_a_forced_older_latest_moves_latest_since(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + + state = _publish(monkeypatch, state, "6.9.0", SECOND, force_latest=True) + + assert state["latest"]["policyengine"] == "6.9.0" + assert state["latest_since"] == SECOND + + +def test_a_state_without_dates_gets_no_wrong_date_for_what_it_already_routes( + monkeypatch, +): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + del state["published_at"] + del state["latest_since"] + + state = _publish(monkeypatch, state, "6.10.0", SECOND) + + assert "published_at" not in state + assert "latest_since" not in state + + state = _publish(monkeypatch, state, "6.10.1", THIRD) + + assert state["published_at"] == {"6.10.1": THIRD} + assert state["latest_since"] == THIRD + + +def test_a_rollback_moves_latest_since(monkeypatch): + state = _publish(monkeypatch, None, "6.10.0", FIRST) + state = _publish(monkeypatch, state, "6.10.1", SECOND) + + restored = routing_state.build_restored_routing_state( + current_state=state, + policyengine_version="6.10.0", + expected_latest="6.10.1", + now=THIRD, + ) + + assert restored["latest"]["policyengine"] == "6.10.0" + assert restored["latest_since"] == THIRD + assert restored["published_at"] == {"6.10.0": FIRST, "6.10.1": SECOND} + + +def test_coerce_keeps_valid_dates_and_drops_malformed_ones(): + coerced = routing_state.coerce_routing_state( + { + "published_at": {"6.10.0": FIRST, "6.9.0": 5, 7: SECOND}, + "latest_since": "", + } + ) + + assert coerced["published_at"] == {"6.10.0": FIRST} + assert "latest_since" not in coerced + + +def test_utc_now_is_an_iso_utc_time(): + assert routing_state.utc_now().endswith("Z") + assert len(routing_state.utc_now()) == len(FIRST) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py index a3c8db897..1d1fe0f03 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py @@ -50,6 +50,24 @@ def routing_state_bundles(state: dict) -> dict: return bundles if isinstance(bundles, dict) else {} +def routing_state_published_at(state: dict) -> dict[str, str]: + """Return when each policyengine version's route was first published.""" + published_at = state.get("published_at") + if not isinstance(published_at, dict): + return {} + return { + version: timestamp + for version, timestamp in published_at.items() + if isinstance(version, str) and isinstance(timestamp, str) + } + + +def routing_state_latest_since(state: dict) -> str | None: + """Return when latest.policyengine last changed, or None.""" + latest_since = state.get("latest_since") + return latest_since if isinstance(latest_since, str) and latest_since else None + + def _bundle_manifest(state: dict, policyengine_version: str | None) -> dict: if policyengine_version is None: return {} diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py index 53d9b2c36..1aca119f4 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py @@ -23,6 +23,8 @@ SUPPORTED_ROUTE_KINDS, routing_state_bundles, routing_state_latest, + routing_state_latest_since, + routing_state_published_at, routing_state_routes, ) from policyengine_simulation_gateway.routes_common import ( @@ -54,6 +56,8 @@ async def list_versions(request: Request) -> VersionsResponse: us=_version_map_from_state(state, "us"), uk=_version_map_from_state(state, "uk"), bundles=bundle_identities(state), + published_at=routing_state_published_at(state), + latest_since=routing_state_latest_since(state), ) # ``Dict.from_name`` is a lazy handle; the RPCs fire during the diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 715cd2c5a..c7fb43c99 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -2851,6 +2851,26 @@ "type": "object", "title": "Bundles", "description": "Deployed bundle identity keyed by exact PolicyEngine wrapper version, then country: model and data versions, the default dataset and the SHA-256 the bundle records for it (the certified artifact's, or the dataset entry's). `policyengine.latest` is a wrapper version: read its identity at `bundles[latest]`." + }, + "published_at": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Published At", + "description": "When each exact PolicyEngine wrapper version's route was first published, as an ISO 8601 UTC time. Republishing a version keeps its date; versions published before the gateway recorded dates have no entry." + }, + "latest_since": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Latest Since", + "description": "When `policyengine.latest` last changed (a publish that advanced it, a forced latest or a rollback), as an ISO 8601 UTC time; null until a publish records it." } }, "type": "object", diff --git a/projects/policyengine-simulation-gateway/tests/test_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_endpoints.py index 77aad4e46..1e4bf293b 100644 --- a/projects/policyengine-simulation-gateway/tests/test_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_endpoints.py @@ -269,6 +269,8 @@ def test__given_no_active_state__then_versions_fall_back_to_old_dicts( assert response.json() == { "spm_capabilities": {}, "bundles": {}, + "published_at": {}, + "latest_since": None, "policyengine": { "latest": "4.9.0", "4.9.0": "old-py-app", diff --git a/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py b/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py index 790800bea..30eb40900 100644 --- a/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py +++ b/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py @@ -61,6 +61,11 @@ def routing_state() -> dict: "latest": {"policyengine": "6.2.7", "us": "1.700.0", "uk": "2.126.8"}, "routes": {"policyengine": {}, "us": {}, "uk": {}}, "bundles": {}, + "published_at": { + "6.2.6": "2026-09-01T10:00:00Z", + "6.2.7": "2026-09-08T10:00:00Z", + }, + "latest_since": "2026-09-08T10:00:00Z", } for version, (us, uk, sha) in releases.items(): state["routes"]["policyengine"][version] = _app(version) @@ -84,6 +89,8 @@ def test_smoke_fixture_is_the_gateway_versions_response(mock_modal, client): assert body["policyengine"]["latest"] == "6.2.7" assert body["policyengine"]["6.2.7"] == _app("6.2.7") assert body["uk"]["latest"] == "2.126.8" + assert body["latest_since"] == "2026-09-08T10:00:00Z" + assert body["published_at"]["6.2.6"] == "2026-09-01T10:00:00Z" if os.environ.get("REGENERATE_VERSIONS_FIXTURE"): FIXTURE.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") assert json.loads(FIXTURE.read_text()) == body From a93c3c0952c0159a0e0ec936e3397a29caf39fb6 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Tue, 27 Oct 2026 18:06:29 -0400 Subject: [PATCH 2/2] Type route dates as aware datetimes and test the deploy and legacy paths Review follow-up on #961: - The routing state keeps a date only when it is an ISO 8601 time with a timezone, normalized to UTC; the gateway parses the stored text and /versions types published_at and latest_since as AwareDatetime, so the response enforces the format (golden OpenAPI: format date-time). - Tests: the deploy path (plan_routing_publication, record, apply) records the dates it planned; merging the legacy per-kind Dicts keeps the current state's dates; malformed or timezone-less dates are dropped in the state and in /versions. Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 2 +- .../gateway_models.py | 4 +- .../src/modal/routing_publisher/state.py | 32 +++++++++++-- .../version_registry/test_routing_dates.py | 47 ++++++++++++++++++- .../route_resolution.py | 27 ++++++++--- .../tests/golden/openapi.json | 6 ++- .../tests/test_versions_fixture.py | 14 ++++++ 7 files changed, 115 insertions(+), 17 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 094568afd..eb828fe9b 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,7 +1,7 @@ - bump: patch changes: added: - - Recorded in the routing state when each policyengine.py route was first published (published_at, by version) and when latest.policyengine last changed (latest_since, set by a publish that advances it, a forced latest or a rollback), and returned both from the gateway's /versions; a date is recorded only when the publish is that event, so states written before this change show no date for what they already route instead of a later, wrong one, and both fields are optional, so earlier states still read and validate + - Recorded in the routing state when each policyengine.py route was first published (published_at, by version) and when latest.policyengine last changed (latest_since, set by a publish that advances it, a forced latest or a rollback), and returned both from the gateway's /versions; a date is recorded only when the publish is that event, so states written before this change show no date for what they already route instead of a later, wrong one, and both fields are optional, so earlier states still read and validate; only ISO 8601 times with a timezone are kept (normalized to UTC in the state), and /versions types them as timezone-aware datetimes - Added the rule sim.dependencies.uv-version-pinned, checked by a new uv-version quality guard with proof fixtures taken from the unpinned setup-uv step and the ghcr.io/astral-sh/uv:latest copy on main; every astral-sh/setup-uv step, uv container image reference, Modal uv_sync or uv_pip_install call, uv install script or release download, pip, pipx or uv tool install of uv, and uv self update must pin the one uv release written in the new .uv-version file, 0.13.6 - Added the rule sim.quality.python-file-length, checked by a new file-length quality guard with proof fixtures that its test writes to a temporary directory: a tracked Python file has at most 800 lines, or at most 1,500 if it is test code (a file under a tests or fixtures directory, or a conftest.py); the 14 files over their limit on main are listed in a baseline that may only shrink, so the rule starts ratcheted, with a downgrade section, authorized by the maintainer, whose tracking issue closes when the rule is enforced - Added breakdown "local_areas" to the simulation request for UK requests whose region is absent, uk or one of country/england, country/scotland, country/wales and country/northern_ireland, refused with 422 for other regions (including country/ni), other countries, include_cliffs=true, segmented=true or a region_group (include_cliffs=false stays accepted); the job runs one baseline and reform on the local-area dataset alone, with the region's filter, and returns a LocalAreaBreakdownOutput (kind local_areas, model_version, policyengine_version, dataset, data_version, and constituency_impact and local_authority_impact as lists of their own record types), which the job status result now accepts alongside SingleYearMacroOutput diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py index e883d69dd..f1ee159a1 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py @@ -505,7 +505,7 @@ class VersionsResponse(BaseModel): "wrapper version: read its identity at `bundles[latest]`." ), ) - published_at: dict[str, str] = Field( + published_at: dict[str, AwareDatetime] = Field( default_factory=dict, description=( "When each exact PolicyEngine wrapper version's route was first " @@ -514,7 +514,7 @@ class VersionsResponse(BaseModel): "have no entry." ), ) - latest_since: str | None = Field( + latest_since: AwareDatetime | None = Field( default=None, description=( "When `policyengine.latest` last changed (a publish that advanced " diff --git a/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py b/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py index bf905186c..4c850a2bc 100644 --- a/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py +++ b/projects/policyengine-simulation-executor/src/modal/routing_publisher/state.py @@ -62,9 +62,29 @@ class RoutingState(TypedDict): latest_since: NotRequired[str] +_UTC_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + + def utc_now() -> str: """Return the current UTC time as the routing state records it.""" - return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + return datetime.now(UTC).strftime(_UTC_FORMAT) + + +def utc_timestamp(value: object) -> str | None: + """Return ``value`` as the state's UTC time, or None unless it is one. + + Accepts an ISO 8601 time with a timezone and normalizes it to UTC; + anything else (including a time without a timezone) is not a date. + """ + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return parsed.astimezone(UTC).strftime(_UTC_FORMAT) def is_newer_version(candidate: str, current: str | None) -> bool: @@ -138,11 +158,15 @@ def coerce_routing_state(value: object) -> RoutingState: if isinstance(key, str) and isinstance(value, dict) } - published_at = _routing_map(value.get("published_at")) + published_at = { + version: timestamp + for version, raw in _routing_map(value.get("published_at")).items() + if (timestamp := utc_timestamp(raw)) is not None + } if published_at: coerced["published_at"] = published_at - latest_since = value.get("latest_since") - if isinstance(latest_since, str) and latest_since: + latest_since = utc_timestamp(value.get("latest_since")) + if latest_since is not None: coerced["latest_since"] = latest_since return coerced diff --git a/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py b/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py index 3a740e73e..5aa7bd0eb 100644 --- a/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py +++ b/projects/policyengine-simulation-executor/tests/version_registry/test_routing_dates.py @@ -4,8 +4,11 @@ from registry_test_support import ( install_bundle_manifests, + install_routing_store, + publication_plan, versioned_app_name, ) +from src.modal.routing_publisher import legacy as routing_legacy from src.modal.routing_publisher import state as routing_state from src.modal.utils import update_version_registry as registry @@ -117,15 +120,55 @@ def test_a_rollback_moves_latest_since(monkeypatch): def test_coerce_keeps_valid_dates_and_drops_malformed_ones(): coerced = routing_state.coerce_routing_state( { - "published_at": {"6.10.0": FIRST, "6.9.0": 5, 7: SECOND}, + "published_at": { + "6.10.0": FIRST, + "6.10.1": "2026-10-08T11:00:00+02:00", + "6.9.0": 5, + "6.8.0": "last week", + "6.7.0": "2026-10-01T09:00:00", + 7: SECOND, + }, "latest_since": "", } ) - assert coerced["published_at"] == {"6.10.0": FIRST} + # Kept only as ISO 8601 times with a timezone, normalized to UTC; a time + # without a timezone is not a date. + assert coerced["published_at"] == {"6.10.0": FIRST, "6.10.1": SECOND} assert "latest_since" not in coerced +def test_the_deploy_path_records_the_dates_it_plans(monkeypatch, patched_modal): + """The deploy workflow plans a publication, records it, then applies it.""" + original = _publish(monkeypatch, None, "6.10.0", FIRST) + store = install_routing_store(patched_modal, original) + monkeypatch.setattr(routing_state, "utc_now", lambda: SECOND) + + plan = publication_plan(monkeypatch, original, ("6.10.1", *MODELS["6.10.1"])) + registry.record_routing_plan(environment="staging", plan=plan) + registry.apply_routing_plan(environment="staging", plan=plan) + + live = store.get("active") + assert live["published_at"] == {"6.10.0": FIRST, "6.10.1": SECOND} + assert live["latest_since"] == SECOND + + +def test_merging_the_legacy_seed_keeps_the_current_dates(monkeypatch): + current = _publish(monkeypatch, None, "6.10.0", FIRST) + legacy_seed = routing_legacy.build_legacy_seed_routing_state( + policyengine_versions={"6.9.0": versioned_app_name("6.9.0")}, + us_versions={"latest": "2.1.0", "2.1.0": versioned_app_name("6.9.0")}, + uk_versions={"latest": "2.120.0", "2.120.0": versioned_app_name("6.9.0")}, + ) + + merged = routing_legacy.merge_legacy_seed_into_current( + current_state=current, legacy_seed=legacy_seed + ) + + assert merged["published_at"] == {"6.10.0": FIRST} + assert merged["latest_since"] == FIRST + + def test_utc_now_is_an_iso_utc_time(): assert routing_state.utc_now().endswith("Z") assert len(routing_state.utc_now()) == len(FIRST) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py index 1d1fe0f03..d0cb13b8d 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/route_resolution.py @@ -1,6 +1,7 @@ """Resolve a request's Modal app from the active routing state.""" from dataclasses import dataclass +from datetime import datetime SUPPORTED_ROUTE_KINDS = ("policyengine", "us", "uk") @@ -50,22 +51,36 @@ def routing_state_bundles(state: dict) -> dict: return bundles if isinstance(bundles, dict) else {} -def routing_state_published_at(state: dict) -> dict[str, str]: +def _aware_datetime(value: object) -> datetime | None: + """Return ``value`` as a timezone-aware datetime, or None unless it is one. + + The routing state is read from a Modal Dict, so a stored date is only text + until it parses as an ISO 8601 time with a timezone. + """ + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else None + + +def routing_state_published_at(state: dict) -> dict[str, datetime]: """Return when each policyengine version's route was first published.""" published_at = state.get("published_at") if not isinstance(published_at, dict): return {} return { version: timestamp - for version, timestamp in published_at.items() - if isinstance(version, str) and isinstance(timestamp, str) + for version, raw in published_at.items() + if isinstance(version, str) and (timestamp := _aware_datetime(raw)) is not None } -def routing_state_latest_since(state: dict) -> str | None: +def routing_state_latest_since(state: dict) -> datetime | None: """Return when latest.policyengine last changed, or None.""" - latest_since = state.get("latest_since") - return latest_since if isinstance(latest_since, str) and latest_since else None + return _aware_datetime(state.get("latest_since")) def _bundle_manifest(state: dict, policyengine_version: str | None) -> dict: diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index c7fb43c99..444ad3329 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -2854,7 +2854,8 @@ }, "published_at": { "additionalProperties": { - "type": "string" + "type": "string", + "format": "date-time" }, "type": "object", "title": "Published At", @@ -2863,7 +2864,8 @@ "latest_since": { "anyOf": [ { - "type": "string" + "type": "string", + "format": "date-time" }, { "type": "null" diff --git a/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py b/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py index 30eb40900..a1d3747bf 100644 --- a/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py +++ b/projects/policyengine-simulation-gateway/tests/test_versions_fixture.py @@ -137,3 +137,17 @@ def test_versions_reports_the_manifest_wrapper_version(mock_modal, client): mock_modal["dicts"]["simulation-api-routing-state"]["active"] = state bundles = client.get("/versions").json()["bundles"] assert bundles["6.2.6"]["uk"]["policyengine_version"] == "6.2.6.post1" + + +def test_versions_reports_only_dates_with_a_timezone(mock_modal, client): + """A stored date that is not an ISO 8601 time with a timezone is left out.""" + state = routing_state() + state["published_at"] = { + "6.2.6": "2026-09-01T12:00:00+02:00", + "6.2.7": "2026-09-08T10:00:00", + } + state["latest_since"] = "yesterday" + mock_modal["dicts"]["simulation-api-routing-state"]["active"] = state + body = client.get("/versions").json() + assert body["published_at"] == {"6.2.6": "2026-09-01T12:00:00+02:00"} + assert body["latest_since"] is None