diff --git a/changelog.d/636.added.md b/changelog.d/636.added.md new file mode 100644 index 000000000..dd541c118 --- /dev/null +++ b/changelog.d/636.added.md @@ -0,0 +1 @@ +Qualification reads policyengine-scorecard#183's receipts. The vendored replay receipt schema and its mirrors now carry the base's own grid record (`diff.base_events`) and the candidate's missing and unsupported measure-years. A gap counts as known only when the base's record lists the same measure-year as inert, or missing and supported. diff --git a/src/policyengine/data/qualification/replay_policy.md b/src/policyengine/data/qualification/replay_policy.md index 3e5efab05..bb6aa65e6 100644 --- a/src/policyengine/data/qualification/replay_policy.md +++ b/src/policyengine/data/qualification/replay_policy.md @@ -2,38 +2,69 @@ Decided under cos d1344 by workstream W4 on 15 October 2026, after two independent reviews. The pull request that added this file records the -decision, the reviews and the alternatives considered. +decision, the reviews and the alternatives considered. Revised on 16 +October 2026, before first use, after the third to fifth reviews of +policyengine.py#637: scorecard's own record of a mixed known gap no longer +blocks, flag counters must equal the flags the rows carry, and any +candidate value other than £0 at a known gap is a contradiction. `evaluate_replay_receipt` in `policyengine.provenance.qualification` applies this text. Each qualification receipt records the policy id and this file's SHA-256 in its `replay` gate details. The replay gate fails on the regressions a scorecard replay receipt identifies. It calls a gap known only on positive evidence from the -comparison base. It sends every other flagged move to a named reviewer. +comparison base, and passes the rows that record a known gap. It sends +every other flagged move to a named reviewer. ## Fail - A rejected request, `no_supported_years`, integrity `failed` (a crash or missing outputs, persistent or not), or a receipt not bound to the request. -- A mapping flag, whether a flagged row carries it or only the receipt's - counters state it. -- A lost result: a coverage row with a base value and no candidate value. -- A lost response, unless it is a known gap: - - a measure-year the candidate lists as inert; +- A mapping flag. +- A lost result: a coverage row with a base value and no candidate value, + unless it is the record of a known gap (below). +- A lost response: + - a measure-year the candidate lists as inert, unless it is a known gap; - a measure-year total at exactly £0 where the base's absolute value is at least the diff's bound, max(£1m, 1% of the base's absolute value), - whether the base was positive or negative. + whether the base was positive or negative. A known gap never excuses + this: a base whose record lists the measure-year as a gap cannot also + hold a material value for it. - An event whose supported grid is incomplete, unless every inert or - missing measure-year in it is a known gap. A receipt whose integrity is - `incomplete` with no incomplete event also fails. + missing and supported measure-year in it is a known gap. An incomplete + event whose receipt does not list its missing measure-years fails, and so + does one with no inert and no missing and supported measure-year at all. + A receipt whose integrity is `incomplete` with no incomplete event also + fails. +- A receipt that contradicts itself, including a complete event that lists + an inert, or missing and supported, measure-year; flag counters that + differ, either way, from the flags its rows carry; and a candidate value + other than £0 for a measure-year that is a known gap. ## Known gaps -A measure-year is a known gap only when the receipt positively records it -as inert, or missing and supported, in the base, for the same event, -measure and year. Until the receipt carries that base-side record, there -are no known gaps. +A candidate measure-year that is inert, or missing and supported (in the +event's missing list and not its unsupported list), is a known gap only +when the receipt's base-side record for the same event (`diff.base_events`, +taken from the base's verified staging and run manifests) lists the same +measure and year as inert, or as missing and supported, in the base. The +gate records each known gap with the base's staging manifest SHA-256. A +legacy-layout base, or an event the base did not replay, has no record, so +none of its gaps are known. + +A known gap does not block. Where one side is inert and the other missing, +scorecard records the gap as a coverage row with exactly £0 on the inert +side and no value on the other, and an inertness flag on the inert side +only, on a row with no values. Those flags are the gap itself and neither +fail nor send the row to review. (Where the candidate is the inert side, its +inert list already decides that flag.) The gate records every row a known +gap accounts for, in whole or in part, beside the known gaps; the rest of a +partly excused row's flags are judged as usual. Any other row at a known gap +is judged as usual: a coverage row with a base value and no candidate value, +even £0, is still a lost result, and any candidate value other than £0 at a +known gap contradicts the candidate's own record, which lists the +measure-year as inert (exactly £0) or missing (no value). ## Review @@ -48,8 +79,9 @@ review: - support flags (the separate `supported_years` gate fails a candidate that loses a year the previous certificate supported); - coverage gains, coverage rows with no value on either side, and any other - coverage row that is not a lost result; -- inertness flags on a measure-year the candidate does not list as inert; + coverage row that is not a lost result or the record of a known gap; +- inertness flags on a measure-year the candidate does not list as inert, + other than the record of a known gap; - a receipt with no diff against a certified variant. Unsupported years alone never fail. A move below the diff's bound raises diff --git a/src/policyengine/data/schemas/vendor/policyengine-scorecard/README.md b/src/policyengine/data/schemas/vendor/policyengine-scorecard/README.md index b8a52769e..da25727c9 100644 --- a/src/policyengine/data/schemas/vendor/policyengine-scorecard/README.md +++ b/src/policyengine/data/schemas/vendor/policyengine-scorecard/README.md @@ -1,7 +1,9 @@ # Vendored policyengine-scorecard schemas -Copied unchanged from PolicyEngine/policyengine-scorecard, branch -`w6/replay-variants`, commit `dc040de` (`docs/uk_replay/schemas/`): the v1 +Copied unchanged from PolicyEngine/policyengine-scorecard, `main` at commit +`71cad306f37f7f9bfb4b58497ff9ddd78b396e00` (the merge of scorecard#183, which +added `diff.base_events` and the corpus grid lists; +`docs/uk_replay/schemas/`): the v1 pre-promotion replay interface between policyengine.py qualification (W4) and the replay corpus (W6). `policyengine.provenance.qualification` mirrors them, and `tests/test_qualification_contracts.py` checks the mirrors and the diff --git a/src/policyengine/data/schemas/vendor/policyengine-scorecard/replay_receipt.v1.json b/src/policyengine/data/schemas/vendor/policyengine-scorecard/replay_receipt.v1.json index a9d526142..eb18cfa9a 100644 --- a/src/policyengine/data/schemas/vendor/policyengine-scorecard/replay_receipt.v1.json +++ b/src/policyengine/data/schemas/vendor/policyengine-scorecard/replay_receipt.v1.json @@ -364,6 +364,12 @@ } } } + }, + "base_events": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/base_event" + } } } } @@ -614,23 +620,74 @@ "type": "boolean" }, "inert_measure_years": { + "$ref": "#/$defs/measure_years" + }, + "missing_measure_years": { + "$ref": "#/$defs/measure_years" + }, + "unsupported_measure_years": { + "$ref": "#/$defs/measure_years" + } + } + }, + "measure_years": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "measure_key", + "year" + ], + "properties": { + "measure_key": { + "type": "string", + "minLength": 1 + }, + "year": { + "type": "integer" + } + } + }, + "uniqueItems": true + }, + "base_event": { + "type": "object", + "additionalProperties": false, + "required": [ + "staging_manifest_sha256", + "run_manifest_sha256", + "supported_years", + "full_supported_grid_complete", + "inert_measure_years", + "missing_measure_years", + "unsupported_measure_years" + ], + "properties": { + "staging_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "run_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "supported_years": { "type": "array", "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "measure_key", - "year" - ], - "properties": { - "measure_key": { - "type": "string" - }, - "year": { - "type": "integer" - } - } - } + "type": "integer" + }, + "uniqueItems": true + }, + "full_supported_grid_complete": { + "type": "boolean" + }, + "inert_measure_years": { + "$ref": "#/$defs/measure_years" + }, + "missing_measure_years": { + "$ref": "#/$defs/measure_years" + }, + "unsupported_measure_years": { + "$ref": "#/$defs/measure_years" } } } diff --git a/src/policyengine/provenance/qualification.py b/src/policyengine/provenance/qualification.py index dfe383a55..29c505bd7 100644 --- a/src/policyengine/provenance/qualification.py +++ b/src/policyengine/provenance/qualification.py @@ -28,7 +28,9 @@ from __future__ import annotations +import functools import hashlib +import json from collections.abc import Iterable, Mapping, Sequence from typing import Any, ClassVar, Literal, Optional @@ -120,6 +122,41 @@ REPLAY_RELATIVE_BOUND = 0.01 +@functools.cache +def _vendored_validator(name: str) -> Any: + """A validator for one vendored policyengine-scorecard schema.""" + + from importlib.resources import files + + import jsonschema + + schema = json.loads( + ( + files("policyengine") + / "data" + / "schemas" + / "vendor" + / "policyengine-scorecard" + / name + ).read_text(encoding="utf-8") + ) + validator = jsonschema.validators.validator_for(schema) + validator.check_schema(schema) + return validator(schema) + + +def conform_to_vendored_schema(document: Mapping[str, Any], name: str) -> None: + """Raise ``ValueError`` unless ``document`` is valid against the vendored + scorecard schema ``name``. The mirrors are a typed view; the schema decides + what a received document may contain (no nulls where it allows none, no + strings where it requires integers).""" + + error = next(iter(_vendored_validator(name).iter_errors(dict(document))), None) + if error is not None: + where = "/".join(str(part) for part in error.absolute_path) or "the document" + raise ValueError(f"{name}: {error.message} (at {where})") + + def replay_policy_sha256() -> str: """SHA-256 of the replay policy text this package applies.""" @@ -582,11 +619,28 @@ class UnsupportedYear(_Mirror): reason: Literal["before_data_year", "after_extension_end"] -class InertMeasureYear(_Mirror): - measure_key: str +class MeasureYear(_Mirror): + measure_key: str = Field(min_length=1) year: int +#: The name the inert lists used before scorecard#183 shared the definition. +InertMeasureYear = MeasureYear + + +def _unique(values: Optional[Sequence[Any]], name: str) -> None: + """The schema's ``uniqueItems``.""" + + if values is None: + return + keys = [ + value.model_dump_json() if isinstance(value, BaseModel) else repr(value) + for value in values + ] + if len(keys) != len(set(keys)): + raise ValueError(f"{name} lists an item twice.") + + class EventResult(_Mirror): registry_sha256: str = Field(pattern=SHA256_PATTERN) supported_years: list[int] @@ -596,7 +650,47 @@ class EventResult(_Mirror): comparison_provenance_sha256: str = Field(pattern=SHA256_PATTERN) artifacts: int = Field(ge=0) full_supported_grid_complete: bool - inert_measure_years: list[InertMeasureYear] + inert_measure_years: list[MeasureYear] + #: Staging's lists (scorecard#183): the full event grid less what was + #: computed, and the full grid less the supported years. Older receipts + #: omit them. + missing_measure_years: Optional[list[MeasureYear]] = None + unsupported_measure_years: Optional[list[MeasureYear]] = None + + @model_validator(mode="after") + def _unique_items(self) -> EventResult: + for name in ( + "supported_years", + "inert_measure_years", + "missing_measure_years", + "unsupported_measure_years", + ): + _unique(getattr(self, name), name) + return self + + +class BaseEvent(_Mirror): + """The comparison base's own grid record for one event (scorecard#183), + read from the base's staging and run manifests that the diff verified.""" + + staging_manifest_sha256: str = Field(pattern=SHA256_PATTERN) + run_manifest_sha256: str = Field(pattern=SHA256_PATTERN) + supported_years: list[int] + full_supported_grid_complete: bool + inert_measure_years: list[MeasureYear] + missing_measure_years: list[MeasureYear] + unsupported_measure_years: list[MeasureYear] + + @model_validator(mode="after") + def _unique_items(self) -> BaseEvent: + for name in ( + "supported_years", + "inert_measure_years", + "missing_measure_years", + "unsupported_measure_years", + ): + _unique(getattr(self, name), name) + return self class Corpus(_Mirror): @@ -654,6 +748,9 @@ class Diff(_Mirror): diff_sha256: str = Field(pattern=SHA256_PATTERN) flag_counts: FlagCounts flagged_rows: list[FlaggedRow] + #: Per event the base replayed in the current layout (scorecard#183). A + #: legacy-layout base, or an event the base did not replay, has no entry. + base_events: Optional[dict[str, BaseEvent]] = None class ResultsLocation(_Mirror): @@ -692,9 +789,13 @@ class ReplayReceipt(_Mirror): @classmethod def from_document(cls, document: Mapping[str, Any]) -> ReplayReceipt: - """Parse a received receipt, keeping its bytes for digest checks.""" + """Parse a received receipt, keeping its bytes for digest checks. + + It must satisfy both the mirror and the vendored schema. + """ receipt = cls.model_validate(dict(document)) + conform_to_vendored_schema(document, "replay_receipt.v1.json") receipt._received = dict(document) return receipt @@ -759,6 +860,22 @@ def _status_shape(self) -> ReplayReceipt: return self +def _is_the_gap(row: FlaggedRow, gap: Mapping[str, Any]) -> bool: + """Whether a coverage row is scorecard's own record of a mixed known gap. + + Where one side is inert and the other missing, scorecard compares an + inert artifact (identical worlds, a total of exactly £0) with none, and + writes a coverage row with £0 on the inert side and nothing on the + other. Any other shape at a known gap is judged as usual. + """ + + if gap["candidate"] == "inert" and gap["why"] == "missing in the base": + return row.base_gbp is None and row.head_gbp == 0 + if gap["candidate"] == "missing" and gap["why"] == "inert in the base": + return row.head_gbp is None and row.base_gbp == 0 + return False + + def evaluate_replay_receipt( request: ReplayQualificationRequest, status: ReplayStatus, @@ -772,22 +889,30 @@ def evaluate_replay_receipt( (``data/qualification/replay_policy.md``): - fail on rejected or failed replays, mapping flags, lost results - (a base value with no candidate value), lost responses (a measure-year - the candidate lists as inert, or a measure-year total of exactly £0 - against a base at least the diff's bound away) and incomplete grids, - unless a gap is known from base-side evidence; + (a coverage row with a base value and no candidate value), lost + responses (a measure-year total of exactly £0 against a base at least + the diff's bound away, which a known gap never excuses, or a + measure-year the candidate lists as inert, unless it is a known gap) + and incomplete grids (unless every gap in them is known). A gap is + known when the base's own record (``diff.base_events``) lists the same + measure-year as inert, or missing and supported. A known gap does not + block, and neither do the rows by which scorecard records a mixed one + (``_is_the_gap``, and the base-only inertness on a valueless row where + the candidate is missing and the base inert); any candidate value + other than £0 at a known gap fails as a contradiction, and any other + coverage row with a base value and no candidate value, even £0, is a + lost result; - send every other flagged move to review. Unsupported years are reported, never failed on their own. The gate's - details record the policy id, the policy text's SHA-256, the known gaps, - the regressions and the review items. - - The receipt is read as a whole, not by its summary counters: a flag - counts when either ``flag_counts`` or a flagged row carries it, and a - receipt that contradicts itself fails (failures listed under a complete - integrity, more flagged rows of a kind than its count, supported years - with no artifacts, nothing replayed at all) or contradicts the request - (a comparison base other than the ``compare_to`` asked for). + details record the policy id, the policy text's SHA-256, the known gaps + and the rows that record them, the regressions and the review items. + + The receipt is read as a whole: a receipt that contradicts itself fails + (failures listed under a complete integrity, ``flag_counts`` that differ + from its rows' flags, supported years with no artifacts, nothing + replayed at all) or contradicts the request (a comparison base other + than the ``compare_to`` asked for). """ request_id = request.request_id @@ -947,6 +1072,18 @@ def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: replayed = 0 for name, event in sorted(corpus.events.items()): replayed += event.artifacts + listed_missing = { + (item.measure_key, item.year) for item in event.missing_measure_years or () + } - { + (item.measure_key, item.year) + for item in event.unsupported_measure_years or () + } + if event.full_supported_grid_complete and ( + event.inert_measure_years or listed_missing + ): + contradictions.append( + f"{name} is complete but lists inert or missing measure-years" + ) if event.supported_years and event.artifacts == 0: contradictions.append(f"{name} has supported years but no artifacts") if not event.supported_years and not event.unsupported_years: @@ -959,11 +1096,13 @@ def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: for row in rows: for kind in row.flags: row_counts[kind] = row_counts.get(kind, 0) + 1 - for kind, seen in sorted(row_counts.items()): - if counts.get(kind, 0) < seen: + # Scorecard's counters are tallies of its rows' flags, so any difference, + # either way, is a receipt that contradicts itself. + for kind in sorted(set(row_counts) | {kind for kind, n in counts.items() if n}): + seen, counted = row_counts.get(kind, 0), counts.get(kind, 0) + if seen != counted: contradictions.append( - f"{seen} flagged row(s) carry {kind} but flag_counts says " - f"{counts.get(kind, 0)}" + f"{seen} flagged row(s) carry {kind} but flag_counts says {counted}" ) details["flag_counts"] = counts details["flagged_row_counts"] = row_counts @@ -984,14 +1123,55 @@ def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: + "; ".join(contradictions), contradictions=contradictions, ) - # Flags come from every place the receipt can state them, never from the - # summary counters alone: a flag a counter states but no row carries - # still needs review. - present = {kind for kind, n in counts.items() if n} | set(row_counts) - # No receipt yet records the base's own inert and missing measure-years, - # so no gap is known and every inert or incomplete measure-year fails. + # A gap is known only where the base's own grid record (scorecard#183) + # lists the same measure-year as inert, or missing and supported, for the + # same event. A legacy-layout base, or an event the base did not replay, + # has no record, so none of its gaps are known. + base_events = (receipt.diff.base_events or {}) if receipt.diff is not None else {} + + def pairs(items: Optional[Sequence[MeasureYear]]) -> set[tuple[str, int]]: + return {(item.measure_key, item.year) for item in items or ()} + + def missing_supported(record: Any) -> Optional[set[tuple[str, int]]]: + if record.missing_measure_years is None: + return None + return pairs(record.missing_measure_years) - pairs( + record.unsupported_measure_years + ) + known_gaps: list[dict[str, Any]] = [] - known = {(gap["event"], gap["measure_key"], gap["year"]) for gap in known_gaps} + for name, event in sorted(corpus.events.items()): + base = base_events.get(name) + if base is None: + continue + base_inert = pairs(base.inert_measure_years) + base_missing = missing_supported(base) or set() + event_inert = pairs(event.inert_measure_years) + for measure_key, year in sorted( + event_inert | (missing_supported(event) or set()) + ): + if (measure_key, year) in base_inert: + why = "inert in the base" + elif (measure_key, year) in base_missing: + why = "missing in the base" + else: + continue + known_gaps.append( + { + "event": name, + "measure_key": measure_key, + "year": year, + "candidate": "inert" + if (measure_key, year) in event_inert + else "missing", + "why": why, + "base_staging_manifest_sha256": base.staging_manifest_sha256, + } + ) + gaps_at = { + (gap["event"], gap["measure_key"], gap["year"]): gap for gap in known_gaps + } + known = set(gaps_at) listed_inert = set(inert) candidate_inert = {where for where in listed_inert if where not in known} regressions: list[dict[str, Any]] = [] @@ -1024,27 +1204,63 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: ) ) for name in incomplete_events: - regressions.append( - item( - "incomplete", - event=name, - why="the supported grid is incomplete and no gap in it is known", + event = corpus.events[name] + missing = missing_supported(event) + if missing is None: + regressions.append( + item( + "incomplete", + event=name, + why="the supported grid is incomplete and the receipt does " + "not list its missing measure-years", + ) + ) + continue + gaps = pairs(event.inert_measure_years) | missing + unknown = sorted(gap for gap in gaps if (name, *gap) not in known) + if not gaps or unknown: + regressions.append( + item( + "incomplete", + event=name, + why="the supported grid is incomplete and not every inert " + "or missing measure-year in it is a known gap", + measure_years=[list(gap) for gap in unknown], + ) ) - ) if receipt.integrity.status == "incomplete" and not incomplete_events: regressions.append( item("incomplete", why="integrity is incomplete with no incomplete event") ) + gap_rows: list[dict[str, Any]] = [] for row in rows: flags = set(row.flags) where = (row.event, row.measure_key, row.year) + gap = gaps_at.get(where) lost = False + if gap is not None and "coverage" in flags and _is_the_gap(row, gap): + # Scorecard's own expression of a mixed known gap: one side has no + # artifact and the other an inert one (identical worlds, £0). + flags.discard("coverage") if flags & BLOCKING_REPLAY_FLAGS: regressions.append(item("mapping", row)) lost = True if "coverage" in flags and row.base_gbp is not None and row.head_gbp is None: regressions.append(item("lost_result", row)) lost = True + if gap is not None and row.head_gbp not in (None, 0): + # The candidate's record lists the measure-year as inert or + # missing, so no row can hold a candidate value for it (an inert + # artifact's effects are exactly £0; a missing one has none). + regressions.append( + item( + "contradiction", + row, + why="a candidate value for a measure-year its record lists " + "as a gap", + ) + ) + lost = True zero_total = ( row.level == "measure_year" and row.head_gbp == 0 @@ -1052,13 +1268,41 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: and abs(row.base_gbp) >= max(REPLAY_ABSOLUTE_BOUND_GBP, REPLAY_RELATIVE_BOUND * abs(row.base_gbp)) ) - if where in listed_inert and (zero_total or "inertness" in flags): - # The candidate's own inert list decides: a lost response above, - # or a known gap. + if zero_total and where in known: + # A base whose own record lists the measure-year as a gap cannot + # also hold a material value for it: the receipt contradicts + # itself, so the gap does not excuse the loss. + regressions.append( + item( + "lost_response", + row, + why="£0 against a material base value, though the base's " + "record lists the measure-year as a gap", + ) + ) + lost = True + elif where in listed_inert and zero_total: + # The candidate's own inert list decides: a lost response above. lost = True - elif zero_total and where not in known: + elif zero_total: regressions.append(item("lost_response", row, why="£0 against the base")) lost = True + if where in listed_inert or ( + gap is not None + and gap["candidate"] == "missing" + and gap["why"] == "inert in the base" + and row.base_gbp is None + and row.head_gbp is None + ): + # The candidate's inert list decides its inertness (a lost response + # above, or a known gap). Where the candidate is missing and the + # base inert, a base-only inertness flag on a row with no value on + # either side (as scorecard writes it) is the known gap itself. + # The row's other flags are judged as usual. + flags = flags - {"inertness"} + if gap is not None and flags != set(row.flags): + # A row the known gap accounts for, in whole or in part. + gap_rows.append(item("known_gap_row", row)) if not lost and flags & REVIEW_REPLAY_FLAGS: if "coverage" in flags and row.base_gbp is None: kind = ( @@ -1071,17 +1315,12 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: else: kind = "flagged_move" review_items.append(item(kind, row)) - for kind in sorted(present - row_counts.keys()): - # A flag the counters state but no row carries: a blocking kind still - # fails, so a reviewer can never accept it. - (regressions if kind in BLOCKING_REPLAY_FLAGS else review_items).append( - item("counted_flag", flag=kind, count=counts.get(kind, 0)) - ) details.update( { "policy": REPLAY_POLICY_ID, "policy_sha256": replay_policy_sha256(), "known_gaps": known_gaps, + "known_gap_rows": gap_rows, "regressions": regressions, "review_items": review_items, } diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index ecf0dfe35..42b4b7e54 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -64,6 +64,7 @@ ReplayReference, ReplayStatus, ReviewAcceptance, + conform_to_vendored_schema, dominates, evaluate_replay_receipt, qualification_verdict, @@ -937,7 +938,9 @@ def read_replay( raw_status = fetch(state_ref, status_path) if raw_status is None: return replay_gate_while_pending(None), None, None - status = ReplayStatus.model_validate_json(raw_status) + status_document = json.loads(raw_status) + status = ReplayStatus.model_validate(status_document) + conform_to_vendored_schema(status_document, "replay_status.v1.json") if not status.terminal: return replay_gate_while_pending(status), status, None assert status.receipt_commit is not None diff --git a/tests/fixtures/uk_replay/README.md b/tests/fixtures/uk_replay/README.md index 3c15e2c82..98f2d9d6e 100644 --- a/tests/fixtures/uk_replay/README.md +++ b/tests/fixtures/uk_replay/README.md @@ -15,3 +15,12 @@ were copied unchanged from the scorecard's `replay-state` branch at commit `4166508f` (`requests//`). The receipt holds aggregate fiscal totals only. `tests/test_qualification_contracts.py` checks that the replay policy fails it on exactly what the d1344 decision says it should. + +`receipt_6.5.0/` holds the request, terminal status and receipt for the pair +policyengine 6.5.0, policyengine-uk 2.130.1 and uk-data 1.58.1 (request +`d83eaf59…`), copied unchanged from the scorecard's `replay-state` branch +head on 16 October 2026. The status names receipt commit `19a9ccdc` and +`receipt_sha256` `a7a67c26…`; the status at that commit itself still says +`running`, because the terminal status lands one commit later. It is the +first real receipt with `diff.base_events` (scorecard#183), and holds +aggregate fiscal totals only. diff --git a/tests/fixtures/uk_replay/receipt_6.5.0/REPLAY_RECEIPT.json b/tests/fixtures/uk_replay/receipt_6.5.0/REPLAY_RECEIPT.json new file mode 100644 index 000000000..c66460517 --- /dev/null +++ b/tests/fixtures/uk_replay/receipt_6.5.0/REPLAY_RECEIPT.json @@ -0,0 +1,560 @@ +{ + "corpus": { + "corpus_version": "uk-fiscal-event-replay/2", + "events": { + "autumn_budget_2024": { + "artifacts": 30, + "comparison_provenance_sha256": "214c7afe8d9703fed378d0e4386d47abea4c535c3f0b8a2d6ee93bccab822727", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "registry_sha256": "89784d34920fb2d6aae702fb4028f84d7bf87f1694df77a0c9c5d255edbcdd34", + "run_manifest_sha256": "23ab600e88ef0269fa7d73f5e403ec3500e7b0a7d5fcddd60ba14c8239e39fda", + "staging_manifest_sha256": "1d882c1d4eb7a13b523bf99ac31945e2d4a3ebf836d6789565a006d2b71e789e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + }, + "autumn_budget_2025": { + "artifacts": 54, + "comparison_provenance_sha256": "f8b8f13cdf1541f387b474721e2d86efac099ada938165be08e71b9f8a806c04", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "registry_sha256": "3da50f9af6c7986fd5fe453ae2dac34275f319d1f2ccd441e3c438f51a18656e", + "run_manifest_sha256": "413acf6342a0e0a3a277cf1f2e9c0bf3c4be1e103204201f9171b90ab21ff5c5", + "staging_manifest_sha256": "5e4df8946e6cea2fa728200d1af2e6803cb74020cac521dd9b99f8869f80d456", + "supported_years": [ + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + }, + "autumn_statement_2023": { + "artifacts": 20, + "comparison_provenance_sha256": "74b8fba87ffc7a67706e4388009295f28e9da67db15a074c69d9537eb845d699", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "registry_sha256": "197d5583f0171104c673ca1f40b402a77ca2b003f496de37d6bc913fb9fd1021", + "run_manifest_sha256": "eb3705d7edb480a91eb37555c3fd52169861ceee84c5514b5529e248247d882a", + "staging_manifest_sha256": "239703b8c314f555fcaf96adaf573a7831962433fcb0e2b26713a217ec48ec7e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2023": { + "artifacts": 8, + "comparison_provenance_sha256": "c23f0eef316404f78f23f3b7e429bdde59496f5ea2a93b883e20e1bcfb648c2b", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "registry_sha256": "0a78bfe32712bd30bdf9137bdf182e82a266c7da0e9885dd61cac6ef99a76357", + "run_manifest_sha256": "29944c4e0393091d621f260f658f700549c207dda4996dce198907665a2b3baa", + "staging_manifest_sha256": "a298924c7ed24832820ddb95196db1d9e9b7b40d32103762e7a5ac59477ef8fd", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2024": { + "artifacts": 15, + "comparison_provenance_sha256": "e15bb23ddc567b3a5afc3b0b5eeaae8bb08f12a1f17378f64bb7e2b67085011b", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "registry_sha256": "64a3bccfd950e868ffc5d6261bf3903df88a3ebedc9b4444494acc717259817a", + "run_manifest_sha256": "458f2f27eb954422bb8e8a3063b629b43f2ffae9cdf901680924c8846a3e9666", + "staging_manifest_sha256": "3679f6e865bd2068efe46db2ea3358f92d6be9d6ade8d2c1c901901ce9813d07", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_statement_2025": { + "artifacts": 12, + "comparison_provenance_sha256": "01874dea549c740dbb714f5615c45fc090554e47207d06c16641f3b01dfc4703", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "registry_sha256": "d01ce0612f3f24830a8d8ea0abaaa6b2626963316b16750310d303f404fbab5b", + "run_manifest_sha256": "01d97cd339b9a389390e2de1eec9617ab4d3a0e0d3374cd2dd9d466bba717fe4", + "staging_manifest_sha256": "dcd2e1349596fa7cd26140ea2b4d29e3f6a2c2b7744359e93143af98d83bdb6c", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + } + } + }, + "diff": { + "base_events": { + "autumn_budget_2024": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "593a3e37a69faf3c5c4afeda26375aba2e748c208b8193178f6fbef179da6761", + "staging_manifest_sha256": "e9abe69562c9f8c0ccc2973331acb66d5b7d2971be7a22d257fb3a64057f8afb", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [] + }, + "autumn_budget_2025": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "927266db92c873170b3c85617f041a9b37a31343e02543de0ad71285f9f8dabc", + "staging_manifest_sha256": "2f4550aa28e20b1498df5c8d790d751368381ae06a2b2934bb72ec9aa9a7856e", + "supported_years": [ + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "unsupported_measure_years": [] + }, + "autumn_statement_2023": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "run_manifest_sha256": "debb2ffec8011fe577e57ef3e334f72aa0e50d90ab46a8536ab2c231c2259c31", + "staging_manifest_sha256": "f2ed6a429597e7bc42e46398cdb03f69d92511c266762c4bfde6d00cf4beae08", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ] + }, + "spring_budget_2023": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "run_manifest_sha256": "e242bf40a01f186fd6b7c759d8d3927058a5a9e382d08d48df618f28199a6d64", + "staging_manifest_sha256": "7a74c574a781e37e45d5adb05afc694b4ba3fb1665fb45f5630c063ac75bec51", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ] + }, + "spring_budget_2024": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "run_manifest_sha256": "0bce7f6e84de2cec8305ecfc8e769cd2c79e73ec736feb8282e9adaf944a4592", + "staging_manifest_sha256": "161ae8d04e51c4180d1c78ccd5fd21d0fae410f4d9237228bd7c2d9e61b90d37", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ] + }, + "spring_statement_2025": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "e3ad05199bc1285c3f2752db7322652a0c275f4f42a0917425150ce55f094767", + "staging_manifest_sha256": "f663172c6be69557dbe1b533731559e5fa1aacbfd5c9151c030fffca5ebeff93", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [] + } + }, + "base_variant_id": "pe6.4.1_uk2.130.1_policyengine-uk-data-1.58.1", + "diff_sha256": "131405d7b5555dbe0bd148629896c7eb439a5dd0647f723a1315c28188bbe65e", + "flag_counts": { + "construction": 6, + "coverage": 0, + "inertness": 0, + "mapping": 0, + "numerical": 3, + "structural": 0, + "support": 0 + }, + "flagged_rows": [ + { + "base_gbp": 574999000.021347, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 956314309.095192, + "key": "obr_pmd_nov2025:11682:73295721bdfdb9e8", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 574999000.021347, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 956314309.095192, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 574999000.021347, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 956314309.095192, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 2321626476.090164, + "event": "autumn_budget_2024", + "flags": [ + "construction" + ], + "head_gbp": 2321626476.090164, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 2397857290.327587, + "event": "autumn_budget_2024", + "flags": [ + "construction" + ], + "head_gbp": 2397857290.327587, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 2484671863.66201, + "event": "autumn_budget_2024", + "flags": [ + "construction" + ], + "head_gbp": 2484671863.66201, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 2572403121.83226, + "event": "autumn_budget_2024", + "flags": [ + "construction" + ], + "head_gbp": 2572403121.83226, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 2660288740.9552917, + "event": "autumn_budget_2024", + "flags": [ + "construction" + ], + "head_gbp": 2660288740.9552917, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + } + ] + }, + "execution": { + "dataset_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "packaged_bundle_manifest_sha256": "6de668d4f0b417c82d981e334ee5a4b95d3c02904491109e94c62ce84246761c", + "packages": { + "policyengine": { + "record_sha256": "8710127f5a784e9fc0867c34912a03092e6f2b6c3627c046fab6a95d217533b4", + "version": "6.5.0", + "wheel_sha256": "312a04f7ceb7058aa803d012ed836a8fdd0ed4885dea46112f5c205a42193856" + }, + "policyengine-core": { + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine-uk": { + "record_sha256": "365cc5256f85b474bd445403462e17e468a1c2f5da6cc2c2b1cbf626ac20e243", + "version": "2.130.1", + "wheel_sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + } + }, + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python": "3.12.3", + "release_bundle": { + "bundle_id": "uk-6.5.0", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "certified_data_build_id": "policyengine-uk-data-1.58.1", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.1", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": "policyengine-uk", + "model_version": "2.130.1", + "policyengine_version": "6.5.0" + }, + "requirements_sha256": "f37ab7eac78e2b29f2f4b51b2af189ef63517cdda38a7150d60af0dd86f42b99", + "runner": "github-actions Linux X64" + }, + "integrity": { + "failures": [], + "status": "complete" + }, + "receipt_sha256": "a7a67c268c21b983ed47930ea5816a23dcd6b37f75373b1a5fc59883b08f5551", + "rejection": null, + "request_id": "d83eaf590d3ebe846506d9ce9be399fe5b9b95dc58362d573d4e19fd501a649a", + "results": { + "manifest_sha256": "3bd9b294472469aea0b17a57b0fa3ba3c1bde6fa51b6940169b91679d0a76c29", + "path": "requests/d83eaf590d3ebe846506d9ce9be399fe5b9b95dc58362d573d4e19fd501a649a/results/", + "ref": "replay-state", + "repository": "PolicyEngine/policyengine-scorecard" + }, + "schema": "policyengine.uk-replay-receipt/v1", + "status": "complete", + "variant_id": "pe6.5.0_uk2.130.1_policyengine-uk-data-1.58.1", + "variant_identity_sha256": "8d5dcd6ae41f0e7685871806c24de41786f414b8f2079ad62ff8acfb28ed4f3c" +} diff --git a/tests/fixtures/uk_replay/receipt_6.5.0/REQUEST.json b/tests/fixtures/uk_replay/receipt_6.5.0/REQUEST.json new file mode 100644 index 000000000..39ab17e15 --- /dev/null +++ b/tests/fixtures/uk_replay/receipt_6.5.0/REQUEST.json @@ -0,0 +1,59 @@ +{ + "attempt": 1, + "candidate": { + "data": { + "build_id": "policyengine-uk-data-1.58.1", + "dataset": "enhanced_frs_2024_25", + "path": "enhanced_frs_2024_25.h5", + "release_manifest_sha256": "545797bf231cd3681e64b677065d921304e9a32f1e46e757a940a412b4136c17", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "size_bytes": 128409762 + }, + "expected_release_bundle": { + "bundle_id": "uk-6.5.0", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "certified_data_build_id": "policyengine-uk-data-1.58.1", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.1", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": "policyengine-uk", + "model_version": "2.130.1", + "policyengine_version": "6.5.0" + }, + "policyengine": { + "packaged_bundle_manifest_sha256": "6de668d4f0b417c82d981e334ee5a4b95d3c02904491109e94c62ce84246761c", + "source": { + "pypi": { + "wheel_sha256": "312a04f7ceb7058aa803d012ed836a8fdd0ed4885dea46112f5c205a42193856" + } + }, + "version": "6.5.0" + }, + "policyengine_core": { + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine_uk": { + "version": "2.130.1", + "wheel_sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + } + }, + "compare_to": "latest_certified", + "kind": "released_bundle", + "requested_by": { + "commit": "8a30363671e76a4a303bfe806a4223f58c2d4c0f", + "note": "Built by hand by the W4 session on 2026-10-16 for policyengine 6.5.0 (release commit 8a303636, the version production sim-api serves), with released_bundle_request from policyengine.py PR #574 at 61155b5d, because the push workflow's NotifyConsumers request is not on main yet. No workflow run produced it.", + "repository": "PolicyEngine/policyengine.py" + }, + "schema": "policyengine.uk-replay-qualification-request/v1" +} diff --git a/tests/fixtures/uk_replay/receipt_6.5.0/STATUS.json b/tests/fixtures/uk_replay/receipt_6.5.0/STATUS.json new file mode 100644 index 000000000..90904abad --- /dev/null +++ b/tests/fixtures/uk_replay/receipt_6.5.0/STATUS.json @@ -0,0 +1,24 @@ +{ + "history": [ + { + "at": "2026-10-16T05:15:40Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/38854823607", + "status": "queued" + }, + { + "at": "2026-10-16T05:16:00Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/38854853696", + "status": "running" + }, + { + "at": "2026-10-16T05:55:28Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/38854853696", + "status": "complete" + } + ], + "receipt_commit": "19a9ccdc46bcda78a01e81844a838859c4e859d2", + "receipt_sha256": "a7a67c268c21b983ed47930ea5816a23dcd6b37f75373b1a5fc59883b08f5551", + "request_id": "d83eaf590d3ebe846506d9ce9be399fe5b9b95dc58362d573d4e19fd501a649a", + "schema": "policyengine.uk-replay-status/v1", + "status": "complete" +} diff --git a/tests/test_qualification_contracts.py b/tests/test_qualification_contracts.py index 9b55e534a..a81330bd6 100644 --- a/tests/test_qualification_contracts.py +++ b/tests/test_qualification_contracts.py @@ -21,7 +21,9 @@ from __future__ import annotations import copy +import itertools import json +from pathlib import Path import jsonschema import pytest @@ -374,9 +376,146 @@ def _rejected_receipt(): "rejected", lambda r: r.update(integrity={"status": "failed", "failures": []}), ), + ( + "empty inert measure key", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + inert_measure_years=[{"measure_key": "", "year": 2026}] + ), + ), + ( + "inert measure-year twice", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + inert_measure_years=[{"measure_key": "m", "year": 2026}] * 2 + ), + ), + ( + "supported year twice", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + supported_years=[2024, 2024] + ), + ), + ( + "missing measure-year twice", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + missing_measure_years=[{"measure_key": "m", "year": 2026}] * 2 + ), + ), + ( + "base event without its unsupported list", + "complete", + lambda r: r["diff"].update( + base_events={ + "spring_budget_2023": { + k: v + for k, v in _base_event().items() + if k != "unsupported_measure_years" + } + } + ), + ), + ( + "base event with a short digest", + "complete", + lambda r: r["diff"].update( + base_events={ + "spring_budget_2023": { + **_base_event(), + "staging_manifest_sha256": "abc", + } + } + ), + ), + ( + "base event with an extra field", + "complete", + lambda r: r["diff"].update( + base_events={"spring_budget_2023": {**_base_event(), "extra": 1}} + ), + ), + # Round-1 review of #637: the schema allows these keys to be absent, + # never null, and never coerces a string to an integer. + ( + "null missing list", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + missing_measure_years=None + ), + ), + ( + "null unsupported list", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + unsupported_measure_years=None + ), + ), + ( + "null base events", + "complete", + lambda r: r["diff"].update(base_events=None), + ), + ( + "base measure-year as a string", + "complete", + lambda r: r["diff"].update( + base_events={ + "spring_budget_2023": { + **_base_event(), + "inert_measure_years": [{"measure_key": "m", "year": "2026"}], + } + } + ), + ), + ( + "supported year as a string", + "complete", + lambda r: r["corpus"]["events"]["spring_budget_2023"].update( + supported_years=["2024", 2025] + ), + ), ] +def _base_event(inert=(), missing=(), unsupported=()): + """A base-side grid record (scorecard#183) for one event.""" + + def measure_years(items): + return [{"measure_key": m, "year": y} for m, y in items] + + return { + "staging_manifest_sha256": "a" * 64, + "run_manifest_sha256": "b" * 64, + "supported_years": [2024, 2025, 2026, 2027], + "full_supported_grid_complete": not inert + and not set(missing) - set(unsupported), + "inert_measure_years": measure_years(inert), + "missing_measure_years": measure_years(missing), + "unsupported_measure_years": measure_years(unsupported), + } + + +def test_a_receipt_with_base_events_and_grid_lists_is_accepted_by_both(): + """scorecard#183's additive fields, valid: the schema and the mirror both + accept them, and the digest covers them as received.""" + + raw = copy.deepcopy(golden("replay_receipt_golden")) + raw["corpus"]["events"]["spring_budget_2023"].update( + missing_measure_years=[{"measure_key": "m", "year": 2023}], + unsupported_measure_years=[{"measure_key": "m", "year": 2023}], + ) + raw["diff"]["base_events"] = {"spring_budget_2023": _base_event()} + raw = _sealed(raw) + _validator("replay_receipt.v1.json").validate(raw) + receipt = ReplayReceipt.from_document(raw) + assert receipt.computed_receipt_sha256 == raw["receipt_sha256"] + assert receipt.diff.base_events["spring_budget_2023"].run_manifest_sha256 == ( + "b" * 64 + ) + + @pytest.mark.parametrize( ("label", "base", "mutate"), RECEIPT_MUTATIONS, @@ -395,7 +534,8 @@ def test_receipt_mirror_and_schema_reject_the_same_receipts(label, base, mutate) mutate(raw) raw = _sealed(raw) assert not _validator("replay_receipt.v1.json").is_valid(raw), label - with pytest.raises(ValidationError): + # A pydantic ValidationError is a ValueError, as is a schema refusal. + with pytest.raises(ValueError): ReplayReceipt.from_document(raw) @@ -570,7 +710,8 @@ def test_a_measure_year_the_candidate_lists_as_inert_fails(): receipt = _edit( _clean_receipt(), lambda r: r["corpus"]["events"][EVENT].update( - inert_measure_years=[{"measure_key": "uc_childcare", "year": 2026}] + inert_measure_years=[{"measure_key": "uc_childcare", "year": 2026}], + full_supported_grid_complete=False, ), ) gate = _gate(receipt) @@ -578,12 +719,41 @@ def test_a_measure_year_the_candidate_lists_as_inert_fails(): assert gate.details["known_gaps"] == [] +def test_a_complete_event_that_lists_a_gap_contradicts_itself(): + for edit in ( + {"inert_measure_years": [{"measure_key": "uc_childcare", "year": 2026}]}, + { + "missing_measure_years": [{"measure_key": "uc_childcare", "year": 2026}], + "unsupported_measure_years": [], + }, + ): + receipt = _edit( + _clean_receipt(), lambda r, e=edit: r["corpus"]["events"][EVENT].update(e) + ) + gate = _gate(receipt) + assert gate.status == "fail" + assert "complete but lists inert or missing" in gate.summary + + MEASURE = "spring_budget_2023__uc_childcare_cap_increase" -def _receipt_with(rows=(), inert=(), complete=True, integrity="complete", counts=None): +def _receipt_with( + rows=(), + inert=(), + complete=True, + integrity="complete", + counts=None, + missing=None, + unsupported=(), + base_events=None, + sealed=True, +): """The golden receipt with these flagged rows (measure-year totals unless - a row says otherwise), counts that match them, and this event state.""" + a row says otherwise), counts that match them, and this event state. + ``missing`` and ``unsupported`` are the event's grid lists (omitted when + ``missing`` is None, as before scorecard#183); ``base_events`` is + ``diff.base_events``.""" doc = _with_requested_wheels(copy.deepcopy(golden("replay_receipt_golden"))) template = { @@ -605,8 +775,17 @@ def _receipt_with(rows=(), inert=(), complete=True, integrity="complete", counts {"measure_key": measure, "year": year} for measure, year in inert ] event["full_supported_grid_complete"] = complete + if missing is not None: + event["missing_measure_years"] = [ + {"measure_key": measure, "year": year} for measure, year in missing + ] + event["unsupported_measure_years"] = [ + {"measure_key": measure, "year": year} for measure, year in unsupported + ] + if base_events is not None: + doc["diff"]["base_events"] = base_events doc["integrity"]["status"] = integrity - return _sealed(doc) + return _sealed(doc) if sealed else doc # The regression table of the d1344 decision record (second review), plus the @@ -841,25 +1020,1085 @@ def test_the_replay_policy_decides_each_case_of_the_decision_record(name): assert recorded == kinds, (name, recorded) -def test_a_counted_flag_with_no_row_still_needs_review(): - gate = _gate(_receipt_with(counts={"numerical": 1})) +GAP = (MEASURE, 2026) +UNSUPPORTED = (MEASURE, 2023) + + +# Each: candidate inert, candidate missing, candidate unsupported, the base +# events, then the expected status and known-gap reasons. +KNOWN_GAP_CASES = { + "inert in both": ( + [GAP], + [], + [], + {EVENT: _base_event(inert=[GAP])}, + "pass", + ["inert in the base"], + ), + "missing in both": ( + [], + [GAP], + [], + {EVENT: _base_event(missing=[GAP])}, + "pass", + ["missing in the base"], + ), + "inert here, missing in the base": ( + [GAP], + [], + [], + {EVENT: _base_event(missing=[GAP])}, + "pass", + ["missing in the base"], + ), + "the base computed it": ( + [GAP], + [], + [], + {EVENT: _base_event()}, + "fail", + [], + ), + "missing in the base, but unsupported there": ( + [GAP], + [], + [], + {EVENT: _base_event(missing=[GAP], unsupported=[GAP])}, + "fail", + [], + ), + "a legacy base": ( + [GAP], + [], + [], + {}, + "fail", + [], + ), + "a record for another event": ( + [GAP], + [], + [], + {"autumn_budget_2025": _base_event(inert=[GAP])}, + "fail", + [], + ), + "another year inert in the base": ( + [GAP], + [], + [], + {EVENT: _base_event(inert=[(MEASURE, 2027)])}, + "fail", + [], + ), + "incomplete with nothing listed": ( + [], + [UNSUPPORTED], + [UNSUPPORTED], + {EVENT: _base_event()}, + "fail", + [], + ), +} + + +@pytest.mark.parametrize("name", sorted(KNOWN_GAP_CASES)) +def test_a_gap_is_known_only_on_the_bases_own_record(name): + """The decision record's exact test: a candidate measure-year that is + inert, or missing and supported, is a known gap only if the base's record + for that event lists it as inert, or missing and supported.""" + + inert, missing, unsupported, base_events, status, reasons = KNOWN_GAP_CASES[name] + receipt = _receipt_with( + inert=inert, + complete=False, + integrity="incomplete", + missing=missing, + unsupported=unsupported, + base_events=base_events, + ) + _validator("replay_receipt.v1.json").validate(receipt) + gate = _gate(receipt) + assert gate.status == status, (name, gate.summary) + assert [gap["why"] for gap in gate.details["known_gaps"]] == reasons + for gap in gate.details["known_gaps"]: + assert gap["base_staging_manifest_sha256"] == "a" * 64 + assert (gap["event"], gap["measure_key"], gap["year"]) == (EVENT, *GAP) + if status == "fail": + assert {r["kind"] for r in gate.details["regressions"]} & { + "incomplete", + "lost_response", + } + + +OTHER = "spring_budget_2023__other_measure" + +# Round-1 review of #637: eight probes that killed mutants the table missed. +# Each: receipt arguments, the expected status and number of known gaps. +GAP_PROBES = { + "complete, missing only in unsupported years": ( + dict( + complete=True, + integrity="complete", + missing=[UNSUPPORTED], + unsupported=[UNSUPPORTED], + base_events={EVENT: _base_event()}, + ), + "pass", + 0, + ), + "missing here, inert in the base": ( + dict(missing=[GAP], base_events={EVENT: _base_event(inert=[GAP])}), + "pass", + 1, + ), + "another measure inert in the base, same year": ( + dict( + inert=[GAP], + missing=[], + base_events={EVENT: _base_event(inert=[(OTHER, 2026)])}, + ), + "fail", + 0, + ), + "a known inert gap beside a new missing one": ( + dict( + inert=[GAP], + missing=[(MEASURE, 2027)], + base_events={EVENT: _base_event(inert=[GAP])}, + ), + "fail", + 1, + ), + "a known gap against a negative material base": ( + dict( + rows=[{"flags": ["structural"], "base_gbp": -2.5e8, "head_gbp": 0.0}], + inert=[GAP], + missing=[], + base_events={EVENT: _base_event(inert=[GAP])}, + ), + "fail", + 1, + ), + "a known gap against exactly the bound": ( + dict( + rows=[{"flags": ["structural"], "base_gbp": 1e6, "head_gbp": 0.0}], + inert=[GAP], + missing=[], + base_events={EVENT: _base_event(inert=[GAP])}, + ), + "fail", + 1, + ), + "unsupported-only missing here, missing in the base": ( + dict( + missing=[UNSUPPORTED], + unsupported=[UNSUPPORTED], + base_events={EVENT: _base_event(missing=[UNSUPPORTED])}, + ), + "fail", + 0, + ), + "another measure missing in the base, same year": ( + dict(missing=[GAP], base_events={EVENT: _base_event(missing=[(OTHER, 2026)])}), + "fail", + 0, + ), +} + + +@pytest.mark.parametrize("name", sorted(GAP_PROBES)) +def test_the_known_gap_rule_on_the_review_probes(name): + arguments, status, known = GAP_PROBES[name] + arguments = {"complete": False, "integrity": "incomplete", **arguments} + receipt = _receipt_with(**arguments) + _validator("replay_receipt.v1.json").validate(receipt) + gate = _gate(receipt) + assert gate.status == status, (name, gate.summary) + assert len(gate.details["known_gaps"]) == known, name + + +def test_a_known_inert_gap_excuses_only_its_inertness_flag(): + """Round 2 of #637: the row's other flags are still judged.""" + + def gate(flags): + receipt = _receipt_with( + rows=[{"flags": flags, "base_gbp": 0.0, "head_gbp": 0.0}], + inert=[GAP], + complete=False, + integrity="incomplete", + missing=[], + base_events={EVENT: _base_event(inert=[GAP])}, + ) + _validator("replay_receipt.v1.json").validate(receipt) + return _gate(receipt) + + assert gate(["inertness"]).status == "pass" + mixed = gate(["inertness", "construction"]) + assert mixed.status == "needs_review" + assert [item["kind"] for item in mixed.details["review_items"]] == ["flagged_move"] + + +#: The rows scorecard writes for each kind of known gap, from its own +#: compare_artifacts and compare_source_rows (round 3 of #637, executed at +#: scorecard 4d44449): an inert artifact has identical worlds (£0); a missing +#: one has no artifact. (candidate, base) -> rows. +PRODUCER_GAP_ROWS = { + ("inert", "inert"): [], + ("missing", "missing"): [], + ("inert", "missing"): [ + {"flags": ["coverage"], "base_gbp": None, "head_gbp": 0.0}, + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": "source_row", + "key": "obr_row_1", + }, + ], + ("missing", "inert"): [ + {"flags": ["coverage"], "base_gbp": 0.0, "head_gbp": None}, + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": "source_row", + "key": "obr_row_1", + }, + ], +} + + +def _known_gap(rows, candidate="inert", base="inert", second_event=False): + """The gate on a schema-valid, sealed receipt whose one measure-year is a + known gap of this kind, with these flagged rows.""" + + doc = _receipt_with( + rows=rows, + inert=[GAP] if candidate == "inert" else [], + missing=[] if candidate == "inert" else [GAP], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(**{base: [GAP]})}, + sealed=False, + ) + if second_event: + other = copy.deepcopy(doc["corpus"]["events"][EVENT]) + other.update( + inert_measure_years=[], + missing_measure_years=[], + full_supported_grid_complete=True, + ) + doc["corpus"]["events"]["autumn_budget_2025"] = other + doc = _sealed(doc) + _validator("replay_receipt.v1.json").validate(doc) + return _gate(doc) + + +@pytest.mark.parametrize( + ("candidate", "base"), sorted(PRODUCER_GAP_ROWS), ids=lambda side: side +) +def test_every_kind_of_known_gap_passes_as_scorecard_writes_it(candidate, base): + """Round 3 of #637: a known gap does not block (d1344), including the + rows by which scorecard records a mixed one. Before, missing/inert + failed as a lost result and inert/missing went to review.""" + + gate = _known_gap(PRODUCER_GAP_ROWS[candidate, base], candidate, base) + assert gate.status == "pass", gate.summary + (gap,) = gate.details["known_gaps"] + assert (gap["candidate"], gap["why"]) == (candidate, f"{base} in the base") + mixed = candidate != base + assert [row["level"] for row in gate.details["known_gap_rows"]] == ( + ["measure_year", "source_row"] if mixed else [] + ) + + +@pytest.mark.parametrize( + ("candidate", "base", "row", "kind"), + [ + # £0 on the wrong side for the gap's kind: judged as usual. + ("inert", "missing", {"base_gbp": 0.0, "head_gbp": None}, "lost_result"), + ("missing", "inert", {"base_gbp": None, "head_gbp": 0.0}, "coverage_gain"), + # A material base value is still a lost result. + ("missing", "inert", {"base_gbp": 5e8, "head_gbp": None}, "lost_result"), + # A candidate value contradicts the candidate's own record. + ("inert", "missing", {"base_gbp": None, "head_gbp": 5e8}, "contradiction"), + ("missing", "missing", {"base_gbp": None, "head_gbp": 1.0}, "contradiction"), + # inert/inert and missing/missing write no coverage row at all. + ("inert", "inert", {"base_gbp": 0.0, "head_gbp": None}, "lost_result"), + ( + "missing", + "missing", + {"base_gbp": None, "head_gbp": None}, + "coverage_without_values", + ), + ], +) +def test_other_coverage_rows_at_a_known_gap_are_judged_as_usual( + candidate, base, row, kind +): + gate = _known_gap([{"flags": ["coverage"], **row}], candidate, base) + recorded = [r["kind"] for r in gate.details["regressions"]] + [ + r["kind"] for r in gate.details["review_items"] + ] + assert kind in recorded, gate.summary + assert gate.status == ("needs_review" if kind.startswith("coverage") else "fail") + assert gate.details["known_gap_rows"] == [] + + +def test_a_known_gap_row_still_carries_its_other_flags(): + rows = copy.deepcopy(PRODUCER_GAP_ROWS["missing", "inert"]) + rows[0]["flags"].append("construction") + gate = _known_gap(rows, "missing", "inert") + assert gate.status == "needs_review" + assert [r["kind"] for r in gate.details["review_items"]] == ["flagged_move"] + + +# Round 3 of #637: the reach of the inertness exemption. The reviewer's +# probes, kept; one changed with the policy (base-only inertness at a +# missing/inert gap is now the gap itself). +_FLAGS = [ + "numerical", + "structural", + "coverage", + "support", + "inertness", + "construction", + "mapping", +] + + +@pytest.mark.parametrize("n", range(1, len(_FLAGS) + 1)) +def test_a_known_inert_gap_strips_only_inertness_for_every_flag_set(n): + for combo in itertools.combinations(_FLAGS, n): + gate = _known_gap([{"flags": list(combo), "base_gbp": None, "head_gbp": None}]) + rest = set(combo) - {"inertness"} + expected = "fail" if "mapping" in rest else ("needs_review" if rest else "pass") + assert gate.status == expected, combo + assert len(gate.details["known_gaps"]) == 1 + + +@pytest.mark.parametrize( + "where", + [ + {"year": 2027}, + {"measure_key": MEASURE + "_other"}, + {"event": "autumn_budget_2025"}, + ], + ids=["another year", "another measure", "another event"], +) +def test_inertness_beside_a_known_inert_gap_still_needs_review(where): + gate = _known_gap( + [{"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 3e8, **where}], + second_event=True, + ) + assert gate.status == "needs_review", gate.summary + assert [i["kind"] for i in gate.details["review_items"]] == ["base_only_inertness"] + + +def test_inertness_alone_off_the_inert_list_needs_review(): + gate = _gate( + _receipt_with(rows=[{"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 3e8}]) + ) + assert gate.status == "needs_review" + assert [i["kind"] for i in gate.details["review_items"]] == ["base_only_inertness"] + + +@pytest.mark.parametrize( + ("base", "status"), [("inert", "pass"), ("missing", "needs_review")] +) +def test_inertness_on_a_known_missing_gap(base, status): + """Where the base is inert, the base-only inertness is the gap itself. + Where both are missing, an inertness flag contradicts both records and + goes to review.""" + + gate = _known_gap( + [ + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": "source_row", + "key": "obr-row-1", + } + ], + candidate="missing", + base=base, + ) + assert gate.status == status, gate.summary + if status == "needs_review": + assert [i["kind"] for i in gate.details["review_items"]] == [ + "base_only_inertness" + ] + + +@pytest.mark.parametrize("level", ["source_row", "measure_year", "head"]) +def test_a_known_inert_gap_strips_inertness_at_every_level(level): + gate = _known_gap( + [ + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": level, + "key": "k", + } + ] + ) + assert gate.status == "pass", gate.summary + + +def test_a_mixed_inertness_row_off_the_inert_list_is_a_flagged_move(): + gate = _gate( + _receipt_with( + rows=[ + {"flags": ["inertness", "numerical"], "base_gbp": 1e8, "head_gbp": 3e8} + ] + ) + ) + assert [i["kind"] for i in gate.details["review_items"]] == ["flagged_move"] + + +def test_an_unknown_inert_zero_total_is_one_lost_response_and_no_review(): + gate = _gate( + _receipt_with( + rows=[{"flags": ["structural"], "base_gbp": 5e8, "head_gbp": 0.0}], + inert=[GAP], + missing=[], + complete=False, + integrity="incomplete", + ) + ) + assert gate.status == "fail" + assert [r["kind"] for r in gate.details["regressions"]] == [ + "lost_response", + "incomplete", + ] + assert gate.details["review_items"] == [] + + +def test_the_real_6_5_0_receipt(): + """W6's 6.5.0 qualification (request d83eaf59, replay-state 19a9ccdc): + the first real receipt with diff.base_events. Eight flagged rows, all + Autumn Budget 2024's capital gains measure (#186's fiscal-year blend): + review, no regressions, no known gaps.""" + + import json + + path = Path(__file__).parent / "fixtures/uk_replay/receipt_6.5.0" + request = ReplayQualificationRequest.model_validate( + json.loads((path / "REQUEST.json").read_text()) + ) + status = ReplayStatus.model_validate(json.loads((path / "STATUS.json").read_text())) + receipt = ReplayReceipt.from_document( + json.loads((path / "REPLAY_RECEIPT.json").read_text()) + ) + assert ( + receipt.computed_receipt_sha256 + == status.receipt_sha256 + == ("a7a67c268c21b983ed47930ea5816a23dcd6b37f75373b1a5fc59883b08f5551") + ) + assert receipt.diff is not None and len(receipt.diff.base_events or {}) == 6 + gate = evaluate_replay_receipt(request, status, receipt) + assert gate.status == "needs_review", gate.summary + assert gate.details["regressions"] == gate.details["known_gaps"] == [] + items = gate.details["review_items"] + assert [item["kind"] for item in items] == ["flagged_move"] * 8 + assert {item["measure_key"] for item in items} == { + "autumn_budget_2024__capital_gains_main_rates_and_reliefs" + } + assert sorted(item["year"] for item in items) == [ + 2024, + 2024, + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + ] + assert ( + gate.details["base_variant_id"] + == "pe6.4.1_uk2.130.1_policyengine-uk-data-1.58.1" + ) + + +# Round 4 of #637: where the known-gap rules reach. The reviewer's probes. +_BESIDE = [ + {"year": 2027}, + {"measure_key": MEASURE + "_other"}, + {"event": "autumn_budget_2025"}, +] +_BESIDE_IDS = ["another year", "another measure", "another event"] + + +def _kinds(gate): + return ( + [r["kind"] for r in gate.details["regressions"]], + [r["kind"] for r in gate.details["review_items"]], + ) + + +@pytest.mark.parametrize("where", _BESIDE, ids=_BESIDE_IDS) +def test_a_zero_base_coverage_row_beside_a_missing_inert_gap_is_a_lost_result(where): + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": 0.0, "head_gbp": None, **where}], + "missing", + "inert", + second_event=True, + ) + assert gate.status == "fail" and _kinds(gate)[0] == ["lost_result"], gate.summary + assert gate.details["known_gap_rows"] == [] + + +@pytest.mark.parametrize("where", _BESIDE, ids=_BESIDE_IDS) +def test_a_zero_head_coverage_row_beside_an_inert_missing_gap_is_a_gain(where): + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": None, "head_gbp": 0.0, **where}], + "inert", + "missing", + second_event=True, + ) + assert gate.status == "needs_review" and _kinds(gate)[1] == ["coverage_gain"] + assert gate.details["known_gap_rows"] == [] + + +@pytest.mark.parametrize("where", _BESIDE, ids=_BESIDE_IDS) +def test_inertness_beside_a_missing_inert_gap_still_needs_review(where): + row = { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": "source_row", + "key": "obr-row-1", + **where, + } + gate = _known_gap([row], "missing", "inert", second_event=True) assert gate.status == "needs_review" - assert gate.details["review_items"] == [ - {"kind": "counted_flag", "flag": "numerical", "count": 1} + assert _kinds(gate)[1] == ["base_only_inertness"], gate.summary + + +@pytest.mark.parametrize("where", _BESIDE, ids=_BESIDE_IDS) +def test_a_candidate_value_beside_a_gap_is_a_gain_not_a_contradiction(where): + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": None, "head_gbp": 5e8, **where}], + "missing", + "missing", + second_event=True, + ) + assert gate.status == "needs_review" and _kinds(gate)[1] == ["coverage_gain"] + + +@pytest.mark.parametrize( + ("candidate", "base", "row", "status", "kind"), + [ + # exactly £0, not merely immaterial + ( + "inert", + "missing", + {"base_gbp": None, "head_gbp": 0.4}, + "fail", + "contradiction", + ), + ( + "missing", + "inert", + {"base_gbp": 0.4, "head_gbp": None}, + "fail", + "lost_result", + ), + # £0 on both sides is not the gap's shape + ( + "inert", + "missing", + {"base_gbp": 0.0, "head_gbp": 0.0}, + "needs_review", + "flagged_move", + ), + ( + "missing", + "inert", + {"base_gbp": 0.0, "head_gbp": 0.0}, + "needs_review", + "flagged_move", + ), + ], +) +def test_only_exactly_the_gaps_shape_is_excused(candidate, base, row, status, kind): + gate = _known_gap([{"flags": ["coverage"], **row}], candidate, base) + assert gate.status == status, gate.summary + assert kind in _kinds(gate)[0] + _kinds(gate)[1] + assert gate.details["known_gap_rows"] == [] + + +@pytest.mark.parametrize( + ("candidate", "base", "row"), + [ + ("inert", "inert", {"flags": ["numerical"], "base_gbp": 1e8, "head_gbp": 3e8}), + ("missing", "inert", {"flags": ["coverage"], "base_gbp": 0.0, "head_gbp": 5e8}), + ( + "missing", + "missing", + { + "flags": ["construction"], + "base_gbp": None, + "head_gbp": 5e8, + "level": "head", + "key": "income_tax", + }, + ), + ( + "inert", + "missing", + {"flags": ["inertness"], "base_gbp": None, "head_gbp": 5e8}, + ), + ( + "missing", + "inert", + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": 5e8, + "level": "source_row", + "key": "obr-row-1", + }, + ), + ("inert", "inert", {"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 5e8}), + ], +) +def test_any_candidate_value_at_a_known_gap_is_a_contradiction(candidate, base, row): + """Round 4 of #637: the policy says so for every row, not only coverage + rows; before, a missing/inert inertness row with a candidate value passed.""" + + gate = _known_gap([row], candidate, base) + assert gate.status == "fail", gate.summary + assert "contradiction" in _kinds(gate)[0] + assert _kinds(gate)[1] == [] + + +def test_a_candidate_value_contradiction_is_never_also_reviewed(): + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": None, "head_gbp": 5e8}], "inert", "missing" + ) + assert _kinds(gate) == (["contradiction"], []) + + +def test_a_counter_above_a_non_gap_row_is_a_contradiction(): + doc = _receipt_with( + rows=[{"flags": ["numerical"], "base_gbp": 1e8, "head_gbp": 3e8}], + counts={"numerical": 2}, + ) + _validator("replay_receipt.v1.json").validate(doc) + gate = _gate(doc) + assert gate.status == "fail" and _kinds(gate)[0] == ["contradiction"], gate.summary + + +def test_three_producer_shaped_gaps_at_once_pass(): + """Gaps of three kinds in one event, with rows as scorecard writes them.""" + + gaps = { + 2025: ("missing", "inert"), + 2026: ("inert", "missing"), + 2027: ("inert", "inert"), + } + rows = [] + for year, kind in gaps.items(): + rows += [ + {**row, "year": year} for row in copy.deepcopy(PRODUCER_GAP_ROWS[kind]) + ] + doc = _receipt_with( + rows=rows, + inert=[(MEASURE, y) for y, (c, _) in gaps.items() if c == "inert"], + missing=[(MEASURE, y) for y, (c, _) in gaps.items() if c == "missing"], + complete=False, + integrity="incomplete", + base_events={ + EVENT: _base_event( + inert=[(MEASURE, y) for y, (_, b) in gaps.items() if b == "inert"], + missing=[(MEASURE, y) for y, (_, b) in gaps.items() if b == "missing"], + ) + }, + ) + _validator("replay_receipt.v1.json").validate(doc) + gate = _gate(doc) + assert gate.status == "pass", gate.summary + assert len(gate.details["known_gaps"]) == 3 + assert len(gate.details["known_gap_rows"]) == 4 + + +# Round 5 of #637: the reviewer's probes on the strip and the text. +_GAP_KINDS = [ + ("inert", "inert"), + ("missing", "missing"), + ("inert", "missing"), + ("missing", "inert"), +] +_SOURCE_ROW = {"level": "source_row", "key": "obr-row-1"} + + +@pytest.mark.parametrize( + "row", + [ + {"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": None, **_SOURCE_ROW}, + {"flags": ["inertness"], "base_gbp": None, "head_gbp": 0.0, **_SOURCE_ROW}, + {"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 0.0, **_SOURCE_ROW}, + {"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": None}, + {"flags": ["inertness"], "base_gbp": 5e8, "head_gbp": None, **_SOURCE_ROW}, + {"flags": ["inertness"], "base_gbp": -5e8, "head_gbp": None, **_SOURCE_ROW}, + ], + ids=[ + "base £0", + "head £0", + "both £0", + "measure-year base £0", + "base £500m", + "base -£500m", + ], +) +def test_only_a_valueless_inertness_row_is_the_gaps_record(row): + """At a missing/inert gap scorecard flags inertness only on a row with no + value on either side; one with any value, even £0, is judged as usual.""" + + gate = _known_gap([row], "missing", "inert") + assert gate.status == "needs_review", (gate.status, _kinds(gate)) + assert _kinds(gate) == ([], ["base_only_inertness"]) + assert gate.details["known_gap_rows"] == [] + + +def test_the_coverage_record_carrying_inertness_too_is_reviewed(): + gate = _known_gap( + [{"flags": ["coverage", "inertness"], "base_gbp": 0.0, "head_gbp": None}], + "missing", + "inert", + ) + assert gate.status == "needs_review" and _kinds(gate) == ( + [], + ["base_only_inertness"], + ) + + +@pytest.mark.parametrize("base_gbp,head_gbp", [(None, None), (None, 0.0), (0.0, 0.0)]) +def test_inertness_at_an_inert_missing_gap_is_decided_by_the_inert_list( + base_gbp, head_gbp +): + gate = _known_gap( + [ + { + "flags": ["inertness"], + "base_gbp": base_gbp, + "head_gbp": head_gbp, + **_SOURCE_ROW, + } + ], + "inert", + "missing", + ) + assert gate.status == "pass", (gate.status, _kinds(gate)) + + +def test_a_partly_excused_row_is_recorded_and_its_rest_reviewed(): + rows = copy.deepcopy(PRODUCER_GAP_ROWS["missing", "inert"]) + rows[0]["flags"].append("construction") + gate = _known_gap(rows, "missing", "inert") + assert [r["level"] for r in gate.details["known_gap_rows"]] == [ + "measure_year", + "source_row", + ] + assert _kinds(gate) == ([], ["flagged_move"]) + + +def test_a_partly_stripped_inertness_row_is_recorded(): + rows = [ + { + "flags": ["inertness", "numerical"], + "base_gbp": None, + "head_gbp": None, + **_SOURCE_ROW, + } + ] + gate = _known_gap(rows, "missing", "inert") + assert len(gate.details["known_gap_rows"]) == 1 + assert _kinds(gate) == ([], ["flagged_move"]) + + +def test_a_row_the_gap_does_not_account_for_is_not_recorded(): + gate = _known_gap( + [{"flags": ["numerical"], "base_gbp": 0.0, "head_gbp": 0.0}], "inert", "inert" + ) + assert gate.details["known_gap_rows"] == [] + + +@pytest.mark.parametrize( + ("candidate", "base"), [k for k in _GAP_KINDS if k != ("missing", "inert")] +) +@pytest.mark.parametrize("base_gbp", [0.0, 0.4]) +def test_a_base_value_with_no_candidate_value_is_a_lost_result( + candidate, base, base_gbp +): + """Even £0, except as the missing/inert gap's own record (the policy text + said the opposite at £0 until round 5).""" + + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": base_gbp, "head_gbp": None}], + candidate, + base, + ) + assert gate.status == "fail" and "lost_result" in _kinds(gate)[0], ( + gate.status, + _kinds(gate), + ) + + +@pytest.mark.parametrize(("candidate", "base"), _GAP_KINDS) +@pytest.mark.parametrize("head", [5e8, -5e8, 0.4, -1e-9]) +@pytest.mark.parametrize( + ("flags", "level"), + [ + (["inertness"], "source_row"), + (["numerical"], "head"), + (["construction"], "measure_year"), + (["support"], "measure_year"), + (["structural"], "head"), + (["coverage"], "source_row"), + ], +) +def test_any_candidate_value_other_than_zero_at_a_known_gap_fails( + candidate, base, head, flags, level +): + gate = _known_gap( + [ + { + "flags": flags, + "base_gbp": None, + "head_gbp": head, + "level": level, + "key": "k", + } + ], + candidate, + base, + ) + assert gate.status == "fail" and _kinds(gate) == (["contradiction"], []), ( + gate.status, + _kinds(gate), + ) + + +# Round 6 of #637: the reviewer's probes. +@pytest.mark.parametrize( + ("level", "key"), [("source_row", "r"), ("measure_year", "total"), ("head", "v")] +) +def test_a_valueless_inertness_row_at_a_missing_inert_gap_is_the_gap_at_any_level( + level, key +): + gate = _known_gap( + [ + { + "flags": ["inertness"], + "base_gbp": None, + "head_gbp": None, + "level": level, + "key": key, + } + ], + "missing", + "inert", + ) + assert gate.status == "pass" and len(gate.details["known_gap_rows"]) == 1 + + +@pytest.mark.parametrize( + ("candidate", "base"), [("missing", "inert"), ("inert", "missing")] +) +def test_a_valueless_coverage_row_at_a_mixed_gap_is_not_its_record(candidate, base): + """The coverage record is £0 on the inert side; a coverage row with no + value on either side is reviewed, at that gap too.""" + + gate = _known_gap( + [{"flags": ["coverage"], "base_gbp": None, "head_gbp": None}], candidate, base + ) + assert gate.status == "needs_review" + assert _kinds(gate) == ([], ["coverage_without_values"]) + assert gate.details["known_gap_rows"] == [] + + +# Round 2 of #637: combinations of a known gap with other rules, and +# evidence that must stay within its own event. +def test_a_known_gap_never_excuses_a_complete_event_listing_a_gap(): + for inert, missing in (([GAP], []), ([], [GAP])): + gate = _gate( + _receipt_with( + inert=inert, + missing=missing, + base_events={EVENT: _base_event(inert=[GAP])}, + ) + ) + assert gate.status == "fail" + assert [r["kind"] for r in gate.details["regressions"]] == ["contradiction"] + assert gate.details["policy"] == REPLAY_POLICY_ID + + +def test_a_known_gap_never_excuses_a_mapping_flag_or_a_material_lost_result(): + mapping = _gate( + _receipt_with( + rows=[{"flags": ["mapping"], "base_gbp": None, "head_gbp": None}], + inert=[GAP], + missing=[], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(inert=[GAP])}, + ) + ) + assert mapping.status == "fail" and len(mapping.details["known_gaps"]) == 1 + assert [r["kind"] for r in mapping.details["regressions"]] == ["mapping"] + # A base inert by its own record cannot hold £500m: scorecard cannot + # write this row, and the gate still fails it as a lost result. + lost = _gate( + _receipt_with( + rows=[{"flags": ["coverage"], "base_gbp": 5e8, "head_gbp": None}], + missing=[GAP], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(inert=[GAP])}, + ) + ) + assert lost.status == "fail" + assert [r["kind"] for r in lost.details["regressions"]] == ["lost_result"] + + +@pytest.mark.parametrize("candidate", ["inert", "missing"]) +def test_base_evidence_never_crosses_into_another_event(candidate): + """The same measure-year in a second event, which the base did not + replay, is not excused by the first event's record.""" + + receipt = _receipt_with( + inert=[GAP] if candidate == "inert" else [], + missing=[] if candidate == "inert" else [GAP], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(inert=[GAP])}, + sealed=False, + ) + receipt["corpus"]["events"]["autumn_budget_2025"] = copy.deepcopy( + receipt["corpus"]["events"][EVENT] + ) + gate = _gate(_sealed(receipt)) + assert gate.status == "fail" + assert [(gap["event"], gap["candidate"]) for gap in gate.details["known_gaps"]] == [ + (EVENT, candidate) ] + assert {r["event"] for r in gate.details["regressions"]} == {"autumn_budget_2025"} + expected = ( + ["lost_response", "incomplete"] if candidate == "inert" else ["incomplete"] + ) + assert [r["kind"] for r in gate.details["regressions"]] == expected + + +def test_a_known_missing_gap_never_hides_another_missing_measure_year(): + gate = _gate( + _receipt_with( + missing=[GAP, (MEASURE, 2027)], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(missing=[GAP])}, + ) + ) + assert gate.status == "fail" and len(gate.details["known_gaps"]) == 1 + assert gate.details["regressions"][0]["measure_years"] == [[MEASURE, 2027]] + + +def test_an_incomplete_event_without_its_missing_list_fails_even_with_base_evidence(): + """A receipt from before scorecard#183 cannot show what is missing.""" + + receipt = _receipt_with( + inert=[GAP], + complete=False, + integrity="incomplete", + base_events={EVENT: _base_event(inert=[GAP])}, + ) + gate = _gate(receipt) + assert gate.status == "fail" + assert gate.details["known_gaps"][0]["why"] == "inert in the base" + assert [r["kind"] for r in gate.details["regressions"]] == ["incomplete"] + assert "does not list its missing" in gate.details["regressions"][0]["why"] -def test_a_counted_mapping_flag_with_no_row_fails(): - """A blocking flag the counters state still fails, so no reviewer can - accept it.""" +def test_a_known_gap_never_excuses_a_material_base_value(): + """A base whose record lists the measure-year as inert cannot also hold a + material value for it, so a £0 against that value still fails.""" - receipt = _receipt_with(counts={"mapping": 1}) + def receipt(base_gbp): + rows = ( + [] + if base_gbp is None + else [{"flags": ["structural"], "base_gbp": base_gbp, "head_gbp": 0.0}] + ) + return _receipt_with( + rows=rows, + inert=[GAP], + complete=False, + integrity="incomplete", + missing=[], + base_events={EVENT: _base_event(inert=[GAP])}, + ) + + known = _gate(receipt(None)) + assert known.status == "pass" and known.details["regressions"] == [] + contradicted = _gate(receipt(2.5e8)) + assert contradicted.status == "fail" + assert [r["kind"] for r in contradicted.details["regressions"]] == ["lost_response"] + assert "though the base's record" in contradicted.details["regressions"][0]["why"] + + +@pytest.mark.parametrize("kind", ["numerical", "mapping", "inertness"]) +def test_a_counter_with_no_row_is_a_contradiction(kind): + """Scorecard's counters tally its rows' flags (equal on every real + receipt), so a flag only the counters state contradicts the rows: it + fails, whether or not its kind blocks.""" + + receipt = _receipt_with(counts={kind: 1}) _validator("replay_receipt.v1.json").validate(receipt) gate = _gate(receipt) assert gate.status == "fail" - assert gate.details["regressions"] == [ - {"kind": "counted_flag", "flag": "mapping", "count": 1} - ] + assert [r["kind"] for r in gate.details["regressions"]] == ["contradiction"] + assert f"0 flagged row(s) carry {kind} but flag_counts says 1" in gate.summary + + +def test_a_counter_above_its_rows_cannot_hide_behind_a_known_gap(): + """Round 3 of #637: one exempt row with flag_counts.inertness=5 passed; + the excess is a contradiction.""" + + receipt = _receipt_with( + rows=[{"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 0.0}], + inert=[GAP], + missing=[], + complete=False, + integrity="incomplete", + counts={"inertness": 5}, + base_events={EVENT: _base_event(inert=[GAP])}, + ) + gate = _gate(receipt) + assert gate.status == "fail" + assert "1 flagged row(s) carry inertness but flag_counts says 5" in gate.summary + + +@pytest.mark.parametrize( + "path", + [ + Path(__file__).parent / "fixtures/uk_replay/receipt_6.4.1_attempt1", + Path(__file__).parent / "fixtures/uk_replay/receipt_6.5.0", + ], + ids=["6.4.1 attempt 1", "6.5.0"], +) +def test_real_receipts_count_exactly_the_flags_their_rows_carry(path): + import json + + receipt = json.loads((path / "REPLAY_RECEIPT.json").read_text()) + tally: dict[str, int] = {} + for row in receipt["diff"]["flagged_rows"]: + for kind in row["flags"]: + tally[kind] = tally.get(kind, 0) + 1 + assert {k: n for k, n in receipt["diff"]["flag_counts"].items() if n} == tally def test_every_early_failure_carries_the_policy_record(): @@ -1044,6 +2283,70 @@ def oracle(these): assert gate.status == "fail" +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(_ROW, max_size=4), + candidate=st.sampled_from(["inert", "missing"]), + base=st.sampled_from(["inert", "missing", None]), +) +def test_the_replay_policy_with_known_gaps_against_a_reference(rows, candidate, base): + """Round 3 of #637: the policy restated independently, with the + measure-year 2026 inert or missing in the candidate and, unless ``base`` + is None, recorded inert or missing in the base (a known gap). Rows fall + on the gap (2026) or beside it (2025).""" + + kwargs = dict( + inert=[GAP] if candidate == "inert" else [], + missing=[] if candidate == "inert" else [GAP], + complete=False, + integrity="incomplete", + ) + if base is not None: + kwargs["base_events"] = {EVENT: _base_event(**{base: [GAP]})} + gate = _gate(_receipt_with(rows, **kwargs)) + + known = base is not None + fail = not known # an unknown inert or missing measure-year fails + review = False + for row in rows: + flags = set(row["flags"]) + b, h = row["base_gbp"], row["head_gbp"] + at_gap = known and row["year"] == 2026 + if at_gap and "coverage" in flags: + if (candidate, base) == ("inert", "missing") and b is None and h == 0: + flags.discard("coverage") + if (candidate, base) == ("missing", "inert") and h is None and b == 0: + flags.discard("coverage") + if "mapping" in flags: + fail = True + if "coverage" in flags and b is not None and h is None: + fail = True + # Any candidate value at a known gap contradicts the candidate's + # record (round 4 of #637). + if at_gap and h not in (None, 0): + fail = True + if ( + row["level"] == "measure_year" + and h == 0 + and b is not None + and abs(b) >= max(1e6, 0.01 * abs(b)) + ): + fail = True + if (row["year"] == 2026 and candidate == "inert") or ( + at_gap + and (candidate, base) == ("missing", "inert") + and b is None + and h is None + ): + flags.discard("inertness") + if flags - {"mapping"}: + review = True + expected = "fail" if fail else ("needs_review" if review else "pass") + assert gate.status == expected, (candidate, base, rows, gate.summary) + assert (gate.status == "fail") == bool(gate.details["regressions"]) + assert len(gate.details["known_gaps"]) == (1 if known else 0) + + def test_the_comparison_base_must_be_the_one_requested(): raw = copy.deepcopy(golden("qualification_request_golden")) base_variant = _clean_receipt()["diff"]["base_variant_id"] diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index d2d9a81dc..5372f77ae 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -802,6 +802,18 @@ def test_reading_replay_state(): state_ref="replay-state", ) assert gate.status == "fail" and "no REPLAY_RECEIPT.json" in gate.summary + # The mirror accepts an explicit null the status schema forbids; reading + # it refuses rather than trusting it. + with pytest.raises(ValueError, match="replay_status.v1.json"): + read_replay( + request, + fetch=lambda ref, path: ( + json.dumps({**running, "receipt_sha256": None}).encode() + if path.endswith("STATUS.json") + else None + ), + state_ref="replay-state", + ) def test_assembled_receipt_without_replay_is_incomplete():