diff --git a/.github/workflows/certify-uk-release.yaml b/.github/workflows/certify-uk-release.yaml index d9bf360b9..23f1c04e4 100644 --- a/.github/workflows/certify-uk-release.yaml +++ b/.github/workflows/certify-uk-release.yaml @@ -7,8 +7,10 @@ # mandatory gates on that exact commit, replayed by policyengine-scorecard, # and recorded in the uk-qualification-ledger branch. Promotion is # serialized: it readies the newest promotable candidate PR that advances -# main and closes superseded ones. Merging stays a reviewed step until -# cos d1344 rules on automatic promotion. +# main, closes superseded ones, and (cos d1344) fast-forwards main to its +# qualified commit once every check on it has passed, which succeeds only +# while main is still the commit it was built on; otherwise the catch-up +# requalifies it on current main. # # Triggers: a policyengine-uk publish or uk-data release (repository # dispatch), a two-hourly catch-up, or a manual run. A manual run with @@ -29,6 +31,12 @@ on: data_version: description: uk-data release (default newest) required: false + core_version: + description: policyengine-core version (default the certified one if the model accepts it) + required: false + pair_id: + description: The exact pair to qualify; the run fails if the versions resolve to another + required: false force: description: Qualify even if the ledger already has a terminal receipt type: boolean @@ -104,6 +112,8 @@ jobs: PAYLOAD_VERSION: ${{ github.event.client_payload.version }} INPUT_MODEL: ${{ inputs.model_version }} INPUT_DATA: ${{ inputs.data_version }} + INPUT_CORE: ${{ inputs.core_version }} + INPUT_PAIR_ID: ${{ inputs.pair_id }} INPUT_FORCE: ${{ inputs.force }} run: | model="${INPUT_MODEL:-}" @@ -116,6 +126,8 @@ jobs: args=(resolve --out candidate.json --ledger ledger) [[ -n "$model" ]] && args+=(--model-version "$model") [[ -n "$data" ]] && args+=(--data-version "$data") + [[ -n "${INPUT_CORE:-}" ]] && args+=(--core-version "$INPUT_CORE") + [[ -n "${INPUT_PAIR_ID:-}" ]] && args+=(--pair-id "$INPUT_PAIR_ID") [[ "${INPUT_FORCE:-false}" == "true" ]] && args+=(--force) python scripts/check_uk_candidate.py "${args[@]}" - uses: actions/upload-artifact@v4 @@ -133,6 +145,10 @@ jobs: # promotion can ready a replacement after checking the previous head. group: promote-uk cancel-in-progress: false + permissions: + contents: write + pull-requests: write + actions: write outputs: candidate_commit: ${{ steps.stage.outputs.candidate_commit }} duplicate: ${{ steps.duplicate.outputs.duplicate }} @@ -152,37 +168,39 @@ jobs: ref: ${{ needs.resolve.outputs.base_commit }} token: ${{ steps.app-token.outputs.token }} fetch-depth: 0 - # Another run may have staged this attempt while this one waited for - # the promotion concurrency group; never overwrite its evidence. - - name: Check for an attempt staged by another run - id: duplicate - env: - ATTEMPT: ledger/uk/${{ needs.resolve.outputs.pair_id }}/attempts/${{ needs.resolve.outputs.attempt_key }} - run: | - .github/qualification-ledger.sh checkout ledger - if [[ -d "$ATTEMPT" ]]; then - echo "duplicate=true" >> "$GITHUB_OUTPUT" - echo "Attempt already staged by another run; nothing to do." - else - echo "duplicate=false" >> "$GITHUB_OUTPUT" - fi - git worktree remove --force ledger - uses: actions/download-artifact@v4 - if: steps.duplicate.outputs.duplicate == 'false' with: name: candidate path: ${{ runner.temp }}/candidate - uses: astral-sh/setup-uv@v8.1.0 - if: steps.duplicate.outputs.duplicate == 'false' with: version: ${{ env.UV_VERSION }} - uses: actions/setup-python@v6 - if: steps.duplicate.outputs.duplicate == 'false' with: python-version: ${{ env.PYTHON_VERSION }} - name: Install the base environment - if: steps.duplicate.outputs.duplicate == 'false' run: uv sync --frozen --extra uk --extra dev --python ${{ env.PYTHON_VERSION }} + # Another run may have staged this attempt, or another attempt for the + # pair from another main commit, while this one waited for the + # promotion concurrency group: recheck against the ledger as it is now, + # and never overwrite another run's evidence. + - name: Check for an attempt staged by another run + id: duplicate + env: + ATTEMPT: ledger/uk/${{ needs.resolve.outputs.pair_id }}/attempts/${{ needs.resolve.outputs.attempt_key }} + # still-eligible asks gh whether an earlier attempt's run failed; + # without a token every unexpired failed run would read as live. + GH_TOKEN: ${{ github.token }} + run: | + .github/qualification-ledger.sh checkout ledger + if [[ -d "$ATTEMPT" ]]; then + echo "duplicate=true" >> "$GITHUB_OUTPUT" + echo "Attempt already staged by another run; nothing to do." + else + uv run --no-sync python scripts/check_uk_candidate.py still-eligible \ + --candidate "$RUNNER_TEMP/candidate/candidate.json" --ledger ledger + fi + git worktree remove --force ledger - name: Certify the pair and regenerate pins, lock, TROs and constants if: steps.duplicate.outputs.duplicate == 'false' env: @@ -216,8 +234,12 @@ jobs: run: | # A forced attempt may replace an already qualified head. Revoke # its ready status before exposing any new, unqualified commit. - ready_pr=$(gh pr list --head "$BRANCH" --state open --json number,isDraft \ - --jq '.[] | select(.isDraft == false) | .number') + # Only this repository's pull request to main is the candidate's: + # head=: lists only this repository's, so a fork's + # with the same branch name can neither shadow nor crowd it out. + ready_pr=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \ + -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \ + --paginate --jq '.[] | select(.draft == false) | .number') if [[ -n "$ready_pr" ]]; then gh pr ready "$ready_pr" --undo fi @@ -226,6 +248,15 @@ jobs: -c user.email="qualification@policyengine.org" \ commit -q -m "Certify UK candidate pair ${PAIR_ID:0:12} for qualification" git push --force origin "HEAD:refs/heads/$BRANCH" + # Pull request runs test the candidate merged with main as it was then. + # Promotion needs the commit's own tree tested, so run the code-changes + # workflow on the candidate branch itself. + - name: Test the candidate commit's own tree + if: steps.duplicate.outputs.duplicate == 'false' + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ needs.resolve.outputs.branch }} + run: gh workflow run pr_code_changes.yaml --repo "$GITHUB_REPOSITORY" --ref "$BRANCH" - name: Stage the attempt in the ledger id: stage if: steps.duplicate.outputs.duplicate == 'false' @@ -248,8 +279,13 @@ jobs: Ledger: https://github.com/${GITHUB_REPOSITORY}/tree/${LEDGER_BRANCH}/uk/${PAIR_ID} 🤖 Generated with [Claude Code](https://claude.com/claude-code)" - if gh pr view "$BRANCH" --json number >/dev/null 2>&1; then - gh pr edit "$BRANCH" --body "$body" + # By number: gh looks a branch name up among pull requests a fork's + # can shadow. head=: lists only this repository's. + number=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \ + -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \ + --paginate --jq '.[].number') + if [[ -n "$number" ]]; then + gh pr edit "$number" --body "$body" else gh pr create --draft --base main --head "$BRANCH" --title "$title" --body "$body" fi @@ -364,7 +400,15 @@ jobs: env: GH_TOKEN: ${{ steps.app-token.outputs.token }} BRANCH: ${{ needs.resolve.outputs.branch }} - run: gh pr edit "$BRANCH" --body-file "$RUNNER_TEMP/pr-body.md" + run: | + number=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \ + -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \ + --paginate --jq '.[].number') + if [[ -z "$number" ]]; then + echo "::error::no open pull request from $BRANCH to main" + exit 1 + fi + gh pr edit "$number" --body-file "$RUNNER_TEMP/pr-body.md" catch-up: if: github.event_name == 'schedule' @@ -400,13 +444,16 @@ jobs: from pathlib import Path for line in Path("recoverable.jsonl").read_text().splitlines(): - pair = json.loads(line)["pair"] + entry = json.loads(line) + pair, pair_id = entry["pair"], entry["pair_id"] subprocess.run( [ "gh", "workflow", "run", "certify-uk-release.yaml", "--repo", os.environ["GITHUB_REPOSITORY"], "--ref", "main", "-f", f"model_version={pair['model']['version']}", "-f", f"data_version={pair['data']['release_version']}", + "-f", f"core_version={pair['core']['version']}", + "-f", f"pair_id={pair_id}", "-f", "force=false", ], check=True, @@ -427,6 +474,36 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: python scripts/check_uk_candidate.py refresh-pr-checks --ledger ledger + # Automatic promotion needs the target built on current main and compared + # against main's certificate. A run without force requalifies it once per + # main commit, one at a time; repeated catch-ups are duplicates. + - name: Requalify a promotable target qualified against an earlier certificate + env: + GH_TOKEN: ${{ github.token }} + run: | + python scripts/check_uk_candidate.py requalify --ledger ledger > requalify.jsonl + python - <<'PY' + import json + import os + import subprocess + from pathlib import Path + + for line in Path("requalify.jsonl").read_text().splitlines(): + entry = json.loads(line) + pair, pair_id = entry["pair"], entry["pair_id"] + subprocess.run( + [ + "gh", "workflow", "run", "certify-uk-release.yaml", + "--repo", os.environ["GITHUB_REPOSITORY"], "--ref", "main", + "-f", f"model_version={pair['model']['version']}", + "-f", f"data_version={pair['data']['release_version']}", + "-f", f"core_version={pair['core']['version']}", + "-f", f"pair_id={pair_id}", + "-f", "force=false", + ], + check=True, + ) + PY accept: if: github.event_name == 'workflow_dispatch' && inputs.accept_pair_id != '' @@ -475,10 +552,19 @@ jobs: group: promote-uk cancel-in-progress: false steps: + # Promotion pushes main as the App, so the push starts main's release + # workflow; a push with the job's own token would not. + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} - uses: actions/checkout@v6 with: ref: main fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} - uses: astral-sh/setup-uv@v8.1.0 with: version: ${{ env.UV_VERSION }} @@ -487,12 +573,6 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install policyengine without country models run: uv pip install --system -e . - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Promote the newest qualified pair that advances main env: GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/changelog.d/634.changed.md b/changelog.d/634.changed.md new file mode 100644 index 000000000..b8cf823c5 --- /dev/null +++ b/changelog.d/634.changed.md @@ -0,0 +1 @@ +The UK qualification replay gate applies policy d1344-replay-regressions-v1. It fails lost results, lost responses (a measure-year the candidate lists as inert, or a measure-year total of £0 against a base at least the diff's bound away), mapping flags and incomplete grids. It sends every other flagged move to a named reviewer. Each receipt records the policy id and its text's SHA-256. Automatic UK promotion is on (d1344): a qualified candidate still built on current main is moved onto main, by a push leased on its parent, once every check of its own runs on its own tree has passed. Runs of pull requests from other branches or forks at the same commit are not counted; a candidate whose runs may not all be its own pull request's against main (its branch has another pull request that is open or ran at its commit, whatever that one's base, or its own pull request's base was ever changed) is refused with an error, since GitHub does not record which pull request or base a run is for, and a base renamed into main records no event. Candidate pull requests are found through GitHub's listing of this repository's own branches, which forks' pull requests cannot crowd out, and every write to one goes by its number. Refresh re-runs a candidate's failed, timed-out or cancelled runs up to four attempts within GitHub's 30-day window, and a candidate left waiting on a run it will not re-run fails the promote run with an error naming it. diff --git a/docs/engineering/runbooks/uk-qualification.md b/docs/engineering/runbooks/uk-qualification.md index 89306acbc..41e245ce4 100644 --- a/docs/engineering/runbooks/uk-qualification.md +++ b/docs/engineering/runbooks/uk-qualification.md @@ -60,3 +60,61 @@ some from fewer than 10 records, so never commit, paste or publish it. 4. Read `failing[...]["detail"]` locally. Report only the finding (which condition failed, on which route), never the figures. + +## A candidate stranded before automatic promotion + +Automatic promotion moves main to a qualified candidate only once the newest +run of each workflow and event on the candidate branch, from this repository, +has passed, including a `pr_code_changes.yaml` run dispatched on the branch +itself. Runs of pull requests from other branches or forks at the same +commit do not count. The `refresh-pr-checks` step re-runs a failed, timed-out or +cancelled run until it has had four attempts, within GitHub's 30 days of its +first attempt. + +When the candidate waits on a run that refresh will not re-run, the promote +job fails with `::error:: waits on runs refresh will not re-run`, +naming each run, its result and its attempts. That covers a run at four +attempts, one older than 30 days, and one whose result refresh never re-runs +(`action_required`, `skipped`, `stale`, `startup_failure` and the like). The +job fails on every scheduled run until one of these clears it: + +- Fix the cause, then re-run the named run by hand + (`gh run rerun --repo PolicyEngine/policyengine.py`); a passing + newest attempt clears it. GitHub refuses this for a run whose first + attempt is more than 30 days old. +- For the run on the candidate's own tree, dispatch a new one: + `gh workflow run pr_code_changes.yaml --repo PolicyEngine/policyengine.py --ref `. +- Requalify the pair: run `certify-uk-release` with its versions, `pair_id` + and `force` set. + +### Another pull request from the candidate branch + +Any other pull request from a candidate branch runs the same workflows with +the same commit and branch as the candidate's own pull request, and +GitHub's run records say neither which pull request a run is for nor which +base it tested; a pull request's current base does not settle it, since a +base renamed into main records no event. So while another pull request from +the branch is open, or after one was closed or merged at the candidate's +commit (whatever its base), or when the candidate's own pull request's base +was ever changed, the promote job fails with +`::error:: is not promoted: runs at its commit may have tested another base`, +naming the pull requests, and refresh re-runs none of the candidate's runs. +Close those pull requests, then requalify the pair as above: the new +candidate commit leaves their runs behind, and prepare opens a new pull +request to main if none is open. + +### What promotion trusts + +Promotion guards against mistakes by people who can write to this +repository and against anything people who cannot can do (forks' pull +requests, whatever their branch names, and other pull requests opened from +a candidate branch, which make promotion refuse; one to main that prepare's +lookup adopts is the candidate's own, and promotion refuses it if its base +was ever changed). Whatever else happens, main only +ever becomes the exact qualified commit, built on main as it is at the push, +after the dispatched `pr_code_changes.yaml` run on that commit's own tree +passed. It does not try to resist deliberate disguise by people who can +write here, such as renaming another branch into main to make a run against +another base look like the candidate's: main is not protected, so they +could push it directly. Treat any such change as an incident, not as +something the gate will catch. diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index e88246812..d07fdfe5c 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -82,6 +82,7 @@ previous_certificate, read_replay, replay_dispatch_payload, + requalification_target, resolve_candidate, stage_attempt, write_gate_results, @@ -90,6 +91,8 @@ BUNDLE_PATH = REPO_ROOT / "src" / "policyengine" / "data" / "bundle" / "manifest.json" CONFIG_PATH = REPO_ROOT / "src" / "policyengine" / "data" / "qualification" / "uk.json" TIMEOUT = 60 +#: The pull request workflow that runs the test suite on a candidate. +CODE_CHANGES_WORKFLOW = ".github/workflows/pr_code_changes.yaml" def utc_now() -> str: @@ -216,7 +219,17 @@ def cmd_resolve(args: argparse.Namespace) -> int: hf_file=hf_file, model_version=args.model_version or None, data_version=args.data_version or None, + core_version=args.core_version or None, ) + if args.pair_id and resolved.pair.pair_id != args.pair_id: + # A requalification or recovery names one exact pair. Resolving any + # other (a moved data tag, another core) must not qualify it instead. + print( + f"::error::resolved pair {resolved.pair.pair_id[:12]} " + f"({resolved.pair.order_key}), not the requested {args.pair_id[:12]}" + ) + set_output(action="mismatch", pair_id=resolved.pair.pair_id) + return 1 base_commit = git("rev-parse", "HEAD") resolved_at = utc_now() action, reason = candidate_action( @@ -248,6 +261,7 @@ def cmd_resolve(args: argparse.Namespace) -> int: "branch": candidate_branch(resolved.pair), "action": action, "reason": reason, + "force": bool(args.force), "previous_certificate": previous_certificate(bundle).canonical_payload(), } if action == "qualify" and args.ledger: @@ -271,6 +285,26 @@ def cmd_resolve(args: argparse.Namespace) -> int: return 0 +def cmd_still_eligible(args: argparse.Namespace) -> int: + """Whether a resolved candidate should still be qualified, against the + ledger as it is now: run under the preparation lock, so two runs that + resolved on different main commits never both stage an attempt.""" + + payload = json.loads(Path(args.candidate).read_text()) + action, reason = candidate_action( + CandidatePair.model_validate(payload["pair"]), + json.loads(BUNDLE_PATH.read_text()), + Path(args.ledger), + base_commit=payload["base_commit"], + force=bool(payload.get("force")), + now=utc_now(), + owner_is_failed=owner_run_failed, + ) + print(f"{action}: {reason}") + set_output(duplicate="false" if action == "qualify" else "true") + return 0 + + def cmd_prepare(args: argparse.Namespace) -> int: payload = json.loads(Path(args.candidate).read_text()) prepared = prepare_candidate_commit( @@ -493,79 +527,363 @@ def cmd_recoverable(args: argparse.Namespace) -> int: return 0 -def _open_candidate_branches() -> dict[str, str]: - """Open candidate branches and the commit each one's pull request is at.""" +def cmd_requalify(args: argparse.Namespace) -> int: + """Print the promotion target's pair when it must be requalified: its + qualification compared against an earlier certificate, or was built on + an earlier main commit than the one checked out.""" + + bundle = json.loads(BUNDLE_PATH.read_text()) + problems: list[str] = [] + receipts = latest_receipts(Path(args.ledger), problems=problems) + for problem in problems: + print(f"::warning::{problem}", file=sys.stderr) + pair = requalification_target( + receipts, bundle, Path(args.ledger), git("rev-parse", "HEAD") + ) + if pair is not None: + print(json.dumps({"pair": pair.canonical_payload(), "pair_id": pair.pair_id})) + return 0 + + +#: The most open candidate pull requests the listing reads; gh stops a search +#: at its limit (30 by default), so reaching this one is an error. +PULL_REQUEST_LIMIT = 1000 + + +def _candidate_pull_requests() -> list[dict[str, Any]]: + """Open candidate pull requests to main from this repository's own branches. + + ``head:candidate/uk/`` matches by prefix, and a fork's pull request can + carry the same branch name, so only same-repository ones are candidates + (a listing without the field counts as a fork's). Only pull requests to + main are (round 6 of #635): GitHub allows one open pull request per head + and base, so each candidate branch then has at most one. The search reads + up to ``PULL_REQUEST_LIMIT`` pull requests, forks included, and refuses + to go on if it reaches the limit, since one it did not read could be the + target. + """ + + listing = json.loads( + subprocess.run( + [ + "gh", + "pr", + "list", + "--repo", + LEDGER_REPOSITORY, + "--state", + "open", + "--base", + "main", + "--search", + "head:candidate/uk/", + "--limit", + str(PULL_REQUEST_LIMIT), + "--json", + "number,baseRefName,headRefName,headRefOid,isDraft,isCrossRepository", + ], + check=True, + capture_output=True, + text=True, + ).stdout + ) + if len(listing) >= PULL_REQUEST_LIMIT: + raise SystemExit( + f"::error::{len(listing)} open pull requests match candidate/uk/; " + "one beyond the listing could be the target" + ) + return [ + item + for item in listing + if item.get("isCrossRepository") is False and item.get("baseRefName") == "main" + ] + + +#: Whether a pull request's base was ever changed: GraphQL records a change +#: a person makes as a BaseRefChangedEvent and one GitHub makes when the base +#: branch is merged and deleted as an AutomaticBaseChangeSucceededEvent +#: (round 8 of #635). Its timeline's totalCount ignores the type filter, so +#: the nodes are read. +BASE_CHANGES_QUERY = ( + "query($owner: String!, $name: String!, $number: Int!) { repository(owner:" + " $owner, name: $name) { pullRequest(number: $number) { timelineItems(" + "itemTypes: [BASE_REF_CHANGED_EVENT, AUTOMATIC_BASE_CHANGE_SUCCEEDED_EVENT]," + " first: 1) { nodes { __typename } pageInfo { hasNextPage } } } } }" +) - listing = subprocess.run( + +def _base_changed(number: Any) -> Optional[bool]: + """Whether pull request ``number``'s base was ever changed; None if that + cannot be read.""" + + owner, name = LEDGER_REPOSITORY.split("/") + result = subprocess.run( [ "gh", - "pr", - "list", - "--repo", - LEDGER_REPOSITORY, - "--state", - "open", - "--search", - "head:candidate/uk/", - "--json", - "headRefName,headRefOid", + "api", + "graphql", + "-f", + f"query={BASE_CHANGES_QUERY}", + "-F", + f"owner={owner}", + "-F", + f"name={name}", + "-F", + f"number={number}", + "--jq", + ".data.repository.pullRequest.timelineItems" + " | (.nodes | length > 0) or .pageInfo.hasNextPage", ], - check=True, + check=False, capture_output=True, text=True, - ).stdout - return {item["headRefName"]: item["headRefOid"] for item in json.loads(listing)} + ) + answer = result.stdout.strip() if result.returncode == 0 else "" + return {"true": True, "false": False}.get(answer) + + +def _shared_branch_pull_requests( + branch: str, commit: str, pull_request: str +) -> Optional[list[str]]: + """Why runs at ``commit`` may not all be the candidate's own pull + request's, against main: the pull requests from this repository's + ``branch`` that make them ambiguous; None if they cannot all be read. + + Any other pull request from the candidate branch runs the same workflows + with the same commit, branch and repository as the candidate's own, + ``pull_request`` (round 6 of #635), and GitHub's run records say neither + which pull request started a run nor which base it tested. A pull + request's current base does not settle it either: a base change keeps + the earlier runs (round 7), and a base branch renamed into main records + no event at all (round 9). So every other pull request from the branch + that is open, or was closed or merged at this commit, counts, whatever + its base, and so does the candidate's own if its base was ever changed. + + What it cannot see: a base renamed into main after running at the + commit on the candidate's own pull request, including one prepare's + lookup adopted as the candidate's after the rename (round 10). Renaming + a branch into main is an admin's deliberate act, outside what promotion + guards against (``_promote``). The REST listing filtered by + ``head=:`` holds only this repository's pull requests, + all of them (paginated), so a fork's with the same branch name can + neither shadow nor crowd them out. + """ + owner = LEDGER_REPOSITORY.split("/")[0] + result = subprocess.run( + [ + "gh", + "api", + "-X", + "GET", + f"repos/{LEDGER_REPOSITORY}/pulls", + "-f", + f"head={owner}:{branch}", + "-f", + "state=all", + "-f", + "per_page=100", + "--paginate", + "--jq", + ".[] | {number, state, merged: (.merged_at != null), base: .base.ref," + " head_sha: .head.sha} | tojson", + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + listing = [json.loads(line) for line in result.stdout.splitlines() if line] + except json.JSONDecodeError: + return None + shared = sorted( + f"#{pr.get('number')} (to {pr.get('base')}, " + f"{'merged' if pr.get('merged') else pr.get('state')})" + for pr in listing + if str(pr.get("number")) != pull_request + and (pr.get("state") == "open" or pr.get("head_sha") == commit) + ) + changed = _base_changed(pull_request) + if changed is None: + return None + if changed: + shared.append(f"#{pull_request} (the candidate's own, after a base change)") + return shared -def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: - """Re-evaluate failed PR checks now that their exact receipt is published. - The PR workflow does not subscribe to edits or ready_for_review events. - GITHUB_TOKEN may re-run existing failed runs with actions: write. +#: A run is re-run until it has made this many attempts in all, so a genuine +#: failure is not retried until a flaky test happens to pass. The first +#: attempt of a qualified_exact_pair candidate's runs fails by design (its +#: ledger receipt is published after they start), so this leaves the re-run +#: that sees the receipt and two more for flakes. A candidate whose newest +#: run is still unsuccessful at the cap fails the promote run visibly. +MAX_RUN_ATTEMPTS = 4 + + +def _newest_runs(runs: list[dict[str, Any]]) -> dict[tuple[Any, Any], dict[str, Any]]: + """The newest run of each workflow (``path``) and event.""" + + newest: dict[tuple[Any, Any], dict[str, Any]] = {} + for run in runs: + key = (run.get("path"), run.get("event")) + if key not in newest or run.get("id", 0) > newest[key].get("id", 0): + newest[key] = run + return newest + + +#: The results refresh re-runs. GitHub re-runs a run only within 30 days of +#: its first attempt. +RERUN_CONCLUSIONS = frozenset({"failure", "timed_out", "cancelled"}) +RERUN_WINDOW = datetime.timedelta(days=30) + + +def _now() -> datetime.datetime: + return datetime.datetime.now(datetime.UTC) + + +def _commit_runs(commit: str) -> Optional[list[dict[str, Any]]]: + """Every workflow run GitHub lists at ``commit``; None if unreadable.""" + + result = subprocess.run( + [ + "gh", + "api", + f"repos/{LEDGER_REPOSITORY}/actions/runs?head_sha={commit}&per_page=100", + "--paginate", + "--jq", + ".workflow_runs[] | {id, head_sha, head_branch," + " head_repository: .head_repository.full_name, path, event, status," + " conclusion, check_suite_id, run_attempt, created_at} | tojson", + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + return [json.loads(line) for line in result.stdout.splitlines() if line] + except json.JSONDecodeError: + return None + + +def _own_runs( + runs: list[dict[str, Any]], commit: str, branch: str +) -> list[dict[str, Any]]: + """The runs of this repository's candidate branch at ``commit``. + + The runs listing for a commit also holds runs of any other pull request + at that commit, a fork's included (round 5 of #635); only runs from this + repository on the candidate branch are the candidate's. Runs of another + pull request from the candidate branch itself look the same, so refresh + and promotion first check there is none + (``_shared_branch_pull_requests``). + """ + + return [ + run + for run in runs + if run.get("head_sha") == commit + and run.get("head_branch") == branch + and run.get("head_repository") == LEDGER_REPOSITORY + ] + + +def _candidate_runs(commit: str, branch: str) -> Optional[list[dict[str, Any]]]: + """``_own_runs`` of the runs listed at ``commit``; None if unreadable.""" + + runs = _commit_runs(commit) + return None if runs is None else _own_runs(runs, commit, branch) + + +def _rerun_due(run: dict[str, Any], now: datetime.datetime) -> bool: + """Whether refresh re-runs this finished run: it failed, timed out or was + cancelled, has attempts left, and GitHub will still re-run it.""" + + if run.get("status") != "completed" or run.get("conclusion") not in ( + RERUN_CONCLUSIONS + ): + return False + if int(run.get("run_attempt") or 1) >= MAX_RUN_ATTEMPTS: + return False + created = run.get("created_at") + if isinstance(created, str) and created: + started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00")) + if now - started > RERUN_WINDOW: + return False + return True + + +def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: + """Re-run a candidate's unsuccessful newest runs once its receipt is out. + + The PR workflow does not subscribe to edits or ready_for_review events, + and the run of the code-changes workflow on the candidate's own tree is + dispatched once, by ``prepare``. So for each workflow and event the + newest of the candidate's own runs (``_candidate_runs``) is re-run when + ``_rerun_due``: a failed or timed-out one by its failed jobs, a + cancelled one (a job timeout cancels the run) in full. Older runs a + newer one replaced are left alone, and so is every run of a candidate + whose runs may have tested another base (``_shared_branch_pull_requests``): + promotion refuses it anyway, and re-runs would retest that base and + spend the attempts. GITHUB_TOKEN may + re-run existing runs with actions: write. """ - branches = _open_candidate_branches() + pull_requests = {pr["headRefName"]: pr for pr in _candidate_pull_requests()} + now = _now() for receipt in latest_receipts(Path(args.ledger)): branch = candidate_branch(receipt.pair) head = receipt.harness.git_commit - if not receipt.promotable or branches.get(branch) != head: + pull_request = pull_requests.get(branch) + if ( + not receipt.promotable + or pull_request is None + or pull_request["headRefOid"] != head + ): continue - listing = subprocess.run( - [ - "gh", - "api", - f"repos/{LEDGER_REPOSITORY}/actions/runs" - f"?head_sha={head}&event=pull_request&per_page=100", - "--paginate", - "--jq", - ".workflow_runs[] | {id, head_sha, event, status, conclusion} | tojson", - ], - check=True, - capture_output=True, - text=True, - ).stdout - for line in listing.splitlines(): - run = json.loads(line) - if ( - run["head_sha"] != head - or run["event"] != "pull_request" - or run["status"] != "completed" - or run["conclusion"] not in {"failure", "timed_out"} - ): + shared = _shared_branch_pull_requests(branch, head, str(pull_request["number"])) + if shared is None: + print(f"::warning::could not read the pull requests from {branch}") + continue + if shared: + print( + f"::warning::not re-running {branch}'s runs: they may have " + f"tested another base, from pull requests {', '.join(shared)}" + ) + continue + listed = _candidate_runs(head, branch) + if listed is None: + print(f"::warning::could not read the runs of {branch} at {head}") + continue + runs = [ + run + for run in listed + if run["event"] in ("pull_request", "workflow_dispatch") + ] + for run in _newest_runs(runs).values(): + if not _rerun_due(run, now): continue - subprocess.run( - [ - "gh", - "run", - "rerun", - str(run["id"]), - "--repo", - LEDGER_REPOSITORY, - "--failed", - ], - check=True, + rerun = subprocess.run( + ["gh", "run", "rerun", str(run["id"]), "--repo", LEDGER_REPOSITORY] + + ([] if run["conclusion"] == "cancelled" else ["--failed"]), + check=False, + ) + if rerun.returncode != 0: + # GitHub refuses some re-runs (a run older than 30 days, one + # cancelled before it had jobs); the steps after refresh must + # still run, so this one is reported and the rest go on. + print( + f"::warning::GitHub did not re-run {run['conclusion']} run " + f"{run['id']} ({run.get('path')}, {run['event']}) at {head}" + ) + continue + print( + f"Re-running {run['conclusion']} run {run['id']} " + f"({run.get('path')}, {run['event']}) at {head}" ) - print(f"Re-running failed PR checks in run {run['id']} at {head}") return 0 @@ -576,7 +894,11 @@ def cmd_promote(args: argparse.Namespace) -> int: for problem in problems: # Such a pair is left out of the plan until its files are rederived. print(f"::warning::{problem}") - branches = _open_candidate_branches() if args.apply else {} + pull_requests = _candidate_pull_requests() if args.apply else [] + branches = {pr["headRefName"]: pr["headRefOid"] for pr in pull_requests} + # Every write to a pull request goes by number: gh looks a branch name up + # among a limited listing, which a fork's pull request can shadow. + numbers = {pr["headRefName"]: str(pr["number"]) for pr in pull_requests} plan = plan_promotion(receipts, bundle, branches) print(f"promotion: {plan.reason}") if plan.target is not None: @@ -594,7 +916,7 @@ def cmd_promote(args: argparse.Namespace) -> int: "gh", "pr", "close", - branch, + numbers.get(branch, branch), "--repo", LEDGER_REPOSITORY, "--comment", @@ -602,30 +924,252 @@ def cmd_promote(args: argparse.Namespace) -> int: ], check=False, ) + # Only the plan's target may be ready, and only while _promote finds it + # on current main: any other ready candidate (left by a run that could + # not undo it, or by older code) goes back to draft. + returned = [ + _set_ready(str(pr["number"]), False) + for pr in sorted(pull_requests, key=lambda pr: pr["number"]) + if not pr["isDraft"] and pr["headRefName"] != plan.ready_branch + ] + safe = True if plan.ready_branch is not None and plan.ready_commit is not None: + safe = _promote( + plan.ready_branch, + plan.ready_commit, + automatic=config()["promotion"]["automatic"], + pull_request=numbers.get(plan.ready_branch, plan.ready_branch), + ) + return 0 if all(returned) and safe else 1 + + +def _set_ready(pull_request: str, ready: bool) -> bool: + """Mark a candidate's pull request (a branch or a number) ready or draft; + whether that worked. + + A draft that stays ready could be merged onto a moved main, so a failed + return to draft is reported as an error for the run to fail on. + """ + + done = ( subprocess.run( - ["gh", "pr", "ready", plan.ready_branch, "--repo", LEDGER_REPOSITORY], + ["gh", "pr", "ready", pull_request, "--repo", LEDGER_REPOSITORY] + + ([] if ready else ["--undo"]), check=False, + ).returncode + == 0 + ) + if not done and not ready: + print(f"::error::could not return {pull_request} to draft") + return done + + +def _promote( + branch: str, commit: str, *, automatic: bool, pull_request: Optional[str] = None +) -> bool: + """Make main exactly the qualified commit, or change nothing. + + The qualified commit's parent is the main commit it was built on. Only + while main is still that commit does a merge of the candidate equal the + qualified tree, so only then is its pull request readied; otherwise it + is kept draft (no merge can combine it with other code) and the catch-up + requalifies it on current main. With automatic promotion the pull + request stays draft until every check on the commit's own tree has + passed, and main is then moved to the commit by a push leased on that + parent: if main is anything else at that instant, nothing changes. + GitHub then records the pull request as merged. + + Returns False when a return to draft failed, which leaves a ready pull + request that a merge could combine with other code, when the candidate + waits on runs refresh will not re-run again, or when its runs may have + tested another base (``_shared_branch_pull_requests``); either way the + promote run fails so someone sees it. ``pull_request`` names the + candidate's pull request for readiness changes (its number; the branch + by default). + + What it guards against: mistakes by people who can write to this + repository, and anything people who cannot can do (forks' pull requests, + whatever branch names they use, and other pull requests opened from the + candidate branch, which make promotion refuse; one to main that + prepare's lookup adopts is the candidate's own, and promotion refuses it + if its base was ever changed). Whatever else happens, main + only ever becomes the exact qualified commit, built on main as it is at + the push, after the dispatched run on that commit's own tree passed. + It does not try to resist deliberate disguise by people who can write + here, such as renaming another branch into main or retargeting pull + requests so that a run against another base looks like the candidate's: + main is not protected, so they could push it directly. The pull request + checks it also requires are as trustworthy as those people. + """ + + target = pull_request or branch + + fetched = subprocess.run( + ["git", "fetch", "origin", f"refs/heads/{branch}"], + check=False, + capture_output=True, + text=True, + ) + head = _git_out("rev-parse", "FETCH_HEAD") if fetched.returncode == 0 else "" + parent = _git_out("rev-parse", f"{commit}^") if head == commit else "" + listed = _git_out("ls-remote", "origin", "refs/heads/main").split() + main = listed[0] if listed else "" + if head != commit or not parent or parent != main: + print( + f"not promotable now: {branch} is not the qualified commit " + f"{commit[:12]} built on current main; it is requalified on current main" ) - if config()["promotion"]["automatic"]: - # GitHub refuses the merge if the branch moved off the qualified - # commit between the plan and the merge. - subprocess.run( - [ - "gh", - "pr", - "merge", - plan.ready_branch, - "--repo", - LEDGER_REPOSITORY, - "--auto", - "--squash", - "--match-head-commit", - plan.ready_commit, - ], - check=False, + return _set_ready(target, False) + if not automatic: + _set_ready(target, True) + return True + shared = _shared_branch_pull_requests(branch, commit, target) + if shared is None: + print( + f"automatic promotion waits for {branch}: the pull requests from it " + "could not be read" + ) + return _set_ready(target, False) + if shared: + # Their runs and the candidate's cannot be told apart, and they stay + # at this commit after the pull requests close: only a new candidate + # commit (and, if the candidate's own was one, a new pull request to + # main, which prepare opens) leaves them behind. + print( + f"::error::{branch} is not promoted: runs at its commit may have " + f"tested another base, from pull requests {', '.join(shared)}, and " + "GitHub does not record which pull request or base a run is for; " + "close them and requalify the pair" + ) + _set_ready(target, False) + return False + waiting, stranded = _unfinished_checks(commit, branch) + if waiting: + print(f"automatic promotion waits for {branch}: " + "; ".join(waiting)) + if stranded: + # Refresh will not re-run these again, and nothing else moves + # the candidate while main stays put: say so in the run's result. + print( + f"::error::{branch} waits on runs refresh will not re-run; re-run " + "or re-dispatch them by hand, or requalify the pair: " + + "; ".join(stranded) ) - return 0 + # A pull request a person or an interrupted run left ready goes back + # to draft until every check on its own tree has passed. + return _set_ready(target, False) and not stranded + _set_ready(target, True) + pushed = subprocess.run( + [ + "git", + "push", + f"--force-with-lease=refs/heads/main:{parent}", + "origin", + f"{commit}:refs/heads/main", + ], + check=False, + capture_output=True, + text=True, + ) + if pushed.returncode != 0: + print( + f"automatic promotion skipped: main is no longer {parent[:12]}; " + "the pair is requalified on current main" + ) + return _set_ready(target, False) + print(f"promoted {branch} at {commit[:12]} to main") + return True + + +def _git_out(*args: str) -> str: + result = subprocess.run(["git", *args], check=False, capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else "" + + +def _unfinished_checks(commit: str, branch: str) -> tuple[list[str], list[str]]: + """Why the qualified commit's checks do not yet show its own tree passing, + and which of the runs it waits on refresh will not re-run. + + Pull request runs test the candidate merged with main as it was then; + the run of the code-changes workflow that the qualification dispatches on + the candidate branch tests the commit's own tree, and must exist. Only + the candidate's own runs count (``_own_runs``): a run of a pull request + from another branch or a fork at the same commit neither blocks nor + passes it (``_promote`` has already refused a candidate whose branch has + another pull request sharing its runs). For each workflow and event, + the newest own run must have + finished successfully, and for each check name the newest check must + have finished as success, skipped or neutral. Checks of any run that is + not a newest own run (one a newer run replaced, or another pull + request's) are not counted at all; checks outside any workflow run (an + app's) are. A run still registering its jobs, no checks, or an + unreadable list waits. A finished, unsuccessful newest own run that + refresh will not re-run (``_rerun_due``) is stranded. + """ + + result = subprocess.run( + [ + "gh", + "api", + f"repos/{LEDGER_REPOSITORY}/commits/{commit}/check-runs?per_page=100", + "--paginate", + "--jq", + ".check_runs[] | {id, name, status, conclusion," + " check_suite_id: .check_suite.id} | tojson", + ], + check=False, + capture_output=True, + text=True, + ) + checks: Optional[list[dict[str, Any]]] = None + if result.returncode == 0: + try: + checks = [json.loads(line) for line in result.stdout.splitlines() if line] + except json.JSONDecodeError: + checks = None + runs = _commit_runs(commit) + if checks is None or runs is None: + return ["the checks could not be read"], [] + newest_runs = _newest_runs(_own_runs(runs, commit, branch)) + excluded_suites = {run.get("check_suite_id") for run in runs} - { + run.get("check_suite_id") for run in newest_runs.values() + } + counted = [ + check for check in checks if check.get("check_suite_id") not in excluded_suites + ] + newest_checks: dict[Any, dict[str, Any]] = {} + for check in counted: + name = check.get("name") + if name not in newest_checks or check.get("id", 0) > newest_checks[name].get( + "id", 0 + ): + newest_checks[name] = check + problems = [ + f"workflow {path} ({event}) is {run.get('conclusion') or run.get('status')}" + for (path, event), run in newest_runs.items() + if run.get("status") != "completed" or run.get("conclusion") != "success" + ] + problems += [ + f"check {name} is {check.get('conclusion') or check.get('status')}" + for name, check in newest_checks.items() + if check.get("status") != "completed" + or check.get("conclusion") not in ("success", "skipped", "neutral") + ] + if (CODE_CHANGES_WORKFLOW, "workflow_dispatch") not in newest_runs: + problems.append( + f"no {CODE_CHANGES_WORKFLOW} run on the candidate's own tree yet" + ) + if not counted: + problems.append("no checks reported yet") + now = _now() + stranded = sorted( + f"workflow {path} ({event}) is {run.get('conclusion')} after " + f"{run.get('run_attempt') or 1} attempt(s)" + for (path, event), run in newest_runs.items() + if run.get("status") == "completed" + and run.get("conclusion") != "success" + and not _rerun_due(run, now) + ) + return sorted(problems), stranded def cmd_accept(args: argparse.Namespace) -> int: @@ -658,11 +1202,18 @@ def parser() -> argparse.ArgumentParser: resolve = commands.add_parser("resolve") resolve.add_argument("--model-version") resolve.add_argument("--data-version") + resolve.add_argument("--core-version") + resolve.add_argument("--pair-id") resolve.add_argument("--ledger") resolve.add_argument("--force", action="store_true") resolve.add_argument("--out", required=True) resolve.set_defaults(func=cmd_resolve) + still_eligible = commands.add_parser("still-eligible") + still_eligible.add_argument("--candidate", required=True) + still_eligible.add_argument("--ledger", required=True) + still_eligible.set_defaults(func=cmd_still_eligible) + prepare = commands.add_parser("prepare") prepare.add_argument("--candidate", required=True) prepare.add_argument("--out", default="prepared.json") @@ -709,6 +1260,10 @@ def parser() -> argparse.ArgumentParser: recoverable.add_argument("--ledger", required=True) recoverable.set_defaults(func=cmd_recoverable) + requalify = commands.add_parser("requalify") + requalify.add_argument("--ledger", required=True) + requalify.set_defaults(func=cmd_requalify) + promote = commands.add_parser("promote") promote.add_argument("--ledger", required=True) promote.add_argument("--apply", action="store_true") diff --git a/src/policyengine/data/qualification/replay_policy.md b/src/policyengine/data/qualification/replay_policy.md new file mode 100644 index 000000000..3e5efab05 --- /dev/null +++ b/src/policyengine/data/qualification/replay_policy.md @@ -0,0 +1,57 @@ +# Replay gate policy d1344-replay-regressions-v1 + +Decided under cos d1344 by workstream W4 on 15 October 2026, after two +independent reviews. The pull request that added this file records the +decision, the reviews and the alternatives considered. +`evaluate_replay_receipt` in `policyengine.provenance.qualification` +applies this text. Each qualification receipt records the policy id and +this file's SHA-256 in its `replay` gate details. + +The replay gate fails on the regressions a scorecard replay receipt +identifies. It calls a gap known only on positive evidence from the +comparison base. It sends every other flagged move to a named reviewer. + +## Fail + +- A rejected request, `no_supported_years`, integrity `failed` (a crash or + missing outputs, persistent or not), or a receipt not bound to the + request. +- A mapping flag, whether a flagged row carries it or only the receipt's + counters state it. +- A lost result: a coverage row with a base value and no candidate value. +- A lost response, unless it is a known gap: + - a measure-year the candidate lists as inert; + - a measure-year total at exactly £0 where the base's absolute value is + at least the diff's bound, max(£1m, 1% of the base's absolute value), + whether the base was positive or negative. +- An event whose supported grid is incomplete, unless every inert or + missing measure-year in it is a known gap. A receipt whose integrity is + `incomplete` with no incomplete event also fails. + +## Known gaps + +A measure-year is a known gap only when the receipt positively records it +as inert, or missing and supported, in the base, for the same event, +measure and year. Until the receipt carries that base-side record, there +are no known gaps. + +## Review + +A needs-review verdict blocks automatic promotion. It stays blocked until a +named reviewer accepts that exact qualification receipt. These go to +review: + +- numerical flags; +- structural flags other than a lost response, including sign reversals + and moves from zero to non-zero; +- construction flags; +- support flags (the separate `supported_years` gate fails a candidate that + loses a year the previous certificate supported); +- coverage gains, coverage rows with no value on either side, and any other + coverage row that is not a lost result; +- inertness flags on a measure-year the candidate does not list as inert; +- a receipt with no diff against a certified variant. + +Unsupported years alone never fail. A move below the diff's bound raises +no numerical flag; it is reviewed only if another flag, such as a structural +one, fires on it. diff --git a/src/policyengine/data/qualification/uk.json b/src/policyengine/data/qualification/uk.json index d50e0ebae..0bcb7f038 100644 --- a/src/policyengine/data/qualification/uk.json +++ b/src/policyengine/data/qualification/uk.json @@ -34,7 +34,7 @@ "standard_reforms_file": "standard_reforms.yaml" }, "promotion": { - "automatic": false, - "decision": "cos d1344: automatic promotion and review bounds are methodology calls; until ruled, promotion is a reviewed PR merge." + "automatic": true, + "decision": "cos d1344 (15 October 2026): automatic promotion is on once policyengine.py#574 lands on its gates. The promote job fast-forwards main to the newest promotable candidate's qualified commit once every check on it has passed, which succeeds only while main is still the commit the candidate was built on; otherwise the candidate is requalified on current main. A needs-review receipt is promoted only after a named reviewer accepts that exact receipt." } } diff --git a/src/policyengine/provenance/qualification.py b/src/policyengine/provenance/qualification.py index 90b6085f9..dfe383a55 100644 --- a/src/policyengine/provenance/qualification.py +++ b/src/policyengine/provenance/qualification.py @@ -28,6 +28,7 @@ from __future__ import annotations +import hashlib from collections.abc import Iterable, Mapping, Sequence from typing import Any, ClassVar, Literal, Optional @@ -52,6 +53,7 @@ __all__ = [ "BLOCKING_REPLAY_FLAGS", + "REPLAY_POLICY_ID", "REQUIRED_GATES", "REVIEW_REPLAY_FLAGS", "CandidatePair", @@ -100,12 +102,31 @@ "error": 3, } -#: Replay diff flags that block promotion whatever their size. Proposed -#: policy pending cos d1344; numerical and support flags need review. -BLOCKING_REPLAY_FLAGS: frozenset[str] = frozenset( - {"structural", "coverage", "mapping", "inertness", "construction"} +#: The replay gate's policy (cos d1344): fail on the regressions a receipt +#: identifies, call a gap known only on base-side evidence, and send every +#: other flagged move to a named reviewer. Its text ships as +#: ``data/qualification/replay_policy.md`` and each receipt records its digest. +REPLAY_POLICY_ID = "d1344-replay-regressions-v1" +REPLAY_POLICY_FILE = "replay_policy.md" +#: Flags that fail whatever the row says. +BLOCKING_REPLAY_FLAGS: frozenset[str] = frozenset({"mapping"}) +#: Flags that need review unless the row is a lost result or lost response. +REVIEW_REPLAY_FLAGS: frozenset[str] = frozenset( + {"numerical", "structural", "coverage", "support", "inertness", "construction"} ) -REVIEW_REPLAY_FLAGS: frozenset[str] = frozenset({"numerical", "support"}) +#: The diff's numerical bound: a move needs at least max(this, 1% of the +#: base's absolute value) to be flagged. +REPLAY_ABSOLUTE_BOUND_GBP = 1_000_000.0 +REPLAY_RELATIVE_BOUND = 0.01 + + +def replay_policy_sha256() -> str: + """SHA-256 of the replay policy text this package applies.""" + + from importlib.resources import files + + data = files("policyengine.data.qualification") / REPLAY_POLICY_FILE + return hashlib.sha256(data.read_bytes()).hexdigest() class GateResult(_Contract): @@ -746,18 +767,27 @@ def evaluate_replay_receipt( """The ``replay`` gate for a terminal replay status and its receipt. The receipt must answer this exact request, be the bytes the status - names, and report an execution of exactly the requested artifacts. - Rejected or failed replays, incomplete runs and blocking flags fail the - gate; numerical and support flags need review. Unsupported years are - reported, never failed on their own. + names, and report an execution of exactly the requested artifacts. Its + contents are then judged by policy ``d1344-replay-regressions-v1`` + (``data/qualification/replay_policy.md``): + + - fail on rejected or failed replays, mapping flags, lost results + (a base value with no candidate value), lost responses (a measure-year + the candidate lists as inert, or a measure-year total of exactly £0 + against a base at least the diff's bound away) and incomplete grids, + unless a gap is known from base-side evidence; + - send every other flagged move to review. + + Unsupported years are reported, never failed on their own. The gate's + details record the policy id, the policy text's SHA-256, the known gaps, + the regressions and the review items. The receipt is read as a whole, not by its summary counters: a flag counts when either ``flag_counts`` or a flagged row carries it, and a receipt that contradicts itself fails (failures listed under a complete integrity, more flagged rows of a kind than its count, supported years with no artifacts, nothing replayed at all) or contradicts the request - (a comparison base other than the ``compare_to`` asked for). Inert - measure-years with no inertness flag need review. + (a comparison base other than the ``compare_to`` asked for). """ request_id = request.request_id @@ -779,38 +809,53 @@ def evaluate_replay_receipt( "receipt_commit": status.receipt_commit, "status": receipt.status, "integrity": receipt.integrity.status, + "policy": REPLAY_POLICY_ID, + "policy_sha256": replay_policy_sha256(), + "known_gaps": [], + "regressions": [], + "review_items": [], } - if problems: + + def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: + """A failure decided before the receipt's contents are judged; the + policy record lists it as the regression.""" + return GateResult( gate_id="replay", status="fail", - summary="Replay receipt is not bound to this request: " - + "; ".join(problems), - details={**details, "binding_problems": problems}, + summary=summary, + details={ + **details, + **extra, + "regressions": [{"kind": kind, "why": summary}], + }, + ) + + if problems: + return fail_early( + "unbound", + "Replay receipt is not bound to this request: " + "; ".join(problems), + binding_problems=problems, ) if receipt.status == "rejected": assert receipt.rejection is not None - return GateResult( - gate_id="replay", - status="fail", - summary=f"Replay rejected the candidate ({receipt.rejection.code}): {receipt.rejection.detail}", - details={**details, "rejection": receipt.rejection.document()}, + return fail_early( + "rejected", + f"Replay rejected the candidate ({receipt.rejection.code}): " + f"{receipt.rejection.detail}", + rejection=receipt.rejection.document(), ) execution = receipt.execution if execution is None: codes = sorted({failure.code for failure in receipt.integrity.failures}) - return GateResult( - gate_id="replay", - status="fail", - summary="Replay failed before the candidate was identified: " + return fail_early( + "failed", + "Replay failed before the candidate was identified: " + (", ".join(codes) or "no failure codes"), - details={ - **details, - "integrity_failures": [ - failure.document() for failure in receipt.integrity.failures - ], - }, + integrity_failures=[ + failure.document() for failure in receipt.integrity.failures + ], ) candidate = request.candidate mismatches: list[str] = [] @@ -854,27 +899,18 @@ def evaluate_replay_receipt( if execution.release_bundle != candidate.expected_release_bundle: mismatches.append("release bundle differs from the expected release bundle") if mismatches: - return GateResult( - gate_id="replay", - status="fail", - summary="Replay ran something other than the candidate: " - + "; ".join(mismatches), - details={**details, "execution_mismatches": mismatches}, + return fail_early( + "unbound", + "Replay ran something other than the candidate: " + "; ".join(mismatches), + execution_mismatches=mismatches, ) if receipt.status == "failed" or receipt.integrity.status == "failed": codes = sorted({failure.code for failure in receipt.integrity.failures}) - return GateResult( - gate_id="replay", - status="fail", - summary="Replay integrity failed: " - + (", ".join(codes) or "no failure codes"), - details={ - **details, - "integrity_failures": [ - f.document() for f in receipt.integrity.failures - ], - }, + return fail_early( + "failed", + "Replay integrity failed: " + (", ".join(codes) or "no failure codes"), + integrity_failures=[f.document() for f in receipt.integrity.failures], ) corpus = receipt.corpus assert corpus is not None @@ -901,17 +937,6 @@ def evaluate_replay_receipt( "inert_measure_years": [list(item) for item in inert], } ) - if receipt.integrity.status != "complete" or incomplete_events: - return GateResult( - gate_id="replay", - status="fail", - summary=( - "Replay did not complete every supported measure-year" - + (f" ({', '.join(incomplete_events)})" if incomplete_events else "") - + "; mandatory gates take no partial evidence." - ), - details={**details, "incomplete_events": incomplete_events}, - ) # A receipt whose parts disagree is not evidence of anything. contradictions: list[str] = [] if receipt.integrity.failures: @@ -953,51 +978,133 @@ def evaluate_replay_receipt( f"but the receipt compares with {base}" ) if contradictions: - return GateResult( - gate_id="replay", - status="fail", - summary="Replay receipt contradicts itself or the request: " + return fail_early( + "contradiction", + "Replay receipt contradicts itself or the request: " + "; ".join(contradictions), - details={**details, "contradictions": contradictions}, + contradictions=contradictions, ) # Flags come from every place the receipt can state them, never from the - # summary counters alone. + # summary counters alone: a flag a counter states but no row carries + # still needs review. present = {kind for kind, n in counts.items() if n} | set(row_counts) - blocking = sorted(present & BLOCKING_REPLAY_FLAGS) - review = sorted(present & REVIEW_REPLAY_FLAGS) - - def tally(kinds: Sequence[str]) -> str: - return ", ".join( - f"{kind} {max(counts.get(kind, 0), row_counts.get(kind, 0))}" - for kind in kinds + # No receipt yet records the base's own inert and missing measure-years, + # so no gap is known and every inert or incomplete measure-year fails. + known_gaps: list[dict[str, Any]] = [] + known = {(gap["event"], gap["measure_key"], gap["year"]) for gap in known_gaps} + listed_inert = set(inert) + candidate_inert = {where for where in listed_inert if where not in known} + regressions: list[dict[str, Any]] = [] + review_items: list[dict[str, Any]] = [] + + def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: + record: dict[str, Any] = {"kind": kind} + if row is not None: + record.update( + event=row.event, + measure_key=row.measure_key, + year=row.year, + level=row.level, + key=row.key, + flags=sorted(row.flags), + base_gbp=row.base_gbp, + head_gbp=row.head_gbp, + ) + record.update(fields) + return record + + for event, measure_key, year in sorted(candidate_inert): + regressions.append( + item( + "lost_response", + event=event, + measure_key=measure_key, + year=year, + why="inert in the candidate", + ) + ) + for name in incomplete_events: + regressions.append( + item( + "incomplete", + event=name, + why="the supported grid is incomplete and no gap in it is known", + ) + ) + if receipt.integrity.status == "incomplete" and not incomplete_events: + regressions.append( + item("incomplete", why="integrity is incomplete with no incomplete event") ) + for row in rows: + flags = set(row.flags) + where = (row.event, row.measure_key, row.year) + lost = False + if flags & BLOCKING_REPLAY_FLAGS: + regressions.append(item("mapping", row)) + lost = True + if "coverage" in flags and row.base_gbp is not None and row.head_gbp is None: + regressions.append(item("lost_result", row)) + lost = True + zero_total = ( + row.level == "measure_year" + and row.head_gbp == 0 + and row.base_gbp is not None + and abs(row.base_gbp) + >= max(REPLAY_ABSOLUTE_BOUND_GBP, REPLAY_RELATIVE_BOUND * abs(row.base_gbp)) + ) + if where in listed_inert and (zero_total or "inertness" in flags): + # The candidate's own inert list decides: a lost response above, + # or a known gap. + lost = True + elif zero_total and where not in known: + regressions.append(item("lost_response", row, why="£0 against the base")) + lost = True + if not lost and flags & REVIEW_REPLAY_FLAGS: + if "coverage" in flags and row.base_gbp is None: + kind = ( + "coverage_gain" + if row.head_gbp is not None + else "coverage_without_values" + ) + elif flags == {"inertness"}: + kind = "base_only_inertness" + else: + kind = "flagged_move" + review_items.append(item(kind, row)) + for kind in sorted(present - row_counts.keys()): + # A flag the counters state but no row carries: a blocking kind still + # fails, so a reviewer can never accept it. + (regressions if kind in BLOCKING_REPLAY_FLAGS else review_items).append( + item("counted_flag", flag=kind, count=counts.get(kind, 0)) + ) + details.update( + { + "policy": REPLAY_POLICY_ID, + "policy_sha256": replay_policy_sha256(), + "known_gaps": known_gaps, + "regressions": regressions, + "review_items": review_items, + } + ) - if blocking: + def tally(records: Sequence[dict[str, Any]]) -> str: + kinds: dict[str, int] = {} + for record in records: + kinds[record["kind"]] = kinds.get(record["kind"], 0) + 1 + return ", ".join(f"{kind} {n}" for kind, n in sorted(kinds.items())) + + if regressions: return GateResult( gate_id="replay", status="fail", - summary="Blocking replay flags: " + tally(blocking), + summary="Replay regressions: " + tally(regressions), details=details, ) - if review: + if review_items: return GateResult( gate_id="replay", status="needs_review", - summary="Replay moves need review: " + tally(review), - details=details, - ) - if inert: - # A measure that does nothing in a supported year, with no inertness - # flag against the base: unchanged from the base, but still a measure - # the model does not respond to. A person decides whether it is known. - return GateResult( - gate_id="replay", - status="needs_review", - summary=( - f"{len(inert)} replayed measure-year(s) are inert, with no " - "inertness flag against the base: " - + ", ".join(f"{m} {y}" for _, m, y in inert[:6]) - ), + summary="Replay moves need review: " + tally(review_items), details=details, ) if receipt.diff is None: diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index 6d711cfc8..ecf0dfe35 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -235,8 +235,13 @@ def resolve_candidate( hf_file: Callable[[str, str, str], bytes], model_version: Optional[str] = None, data_version: Optional[str] = None, + core_version: Optional[str] = None, ) -> ResolvedCandidate: - """Resolve the candidate pair from PyPI and Hugging Face.""" + """Resolve the candidate pair from PyPI and Hugging Face. + + ``core_version`` pins the core, as a requalification of an exact pair + must; otherwise the certified core is kept if the model accepts it. + """ model_project = pypi_project(UK_MODEL_PACKAGE) releases = model_project["releases"] @@ -248,9 +253,17 @@ def resolve_candidate( requires_core = core_requirement(model_release["info"].get("requires_dist") or []) core_project = pypi_project(CORE_PACKAGE) - core_version = choose_core_version( - current_core, requires_core, _stable_versions(core_project["releases"]) - ) + if core_version is None: + core_version = choose_core_version( + current_core, requires_core, _stable_versions(core_project["releases"]) + ) + elif core_version not in core_project["releases"]: + raise ValueError(f"{CORE_PACKAGE} {core_version} is not on PyPI.") + elif Version(core_version) not in SpecifierSet(requires_core): + raise ValueError( + f"{UK_MODEL_PACKAGE} {model_version} requires {CORE_PACKAGE} " + f"{requires_core}, not {core_version}." + ) core_wheel = _wheel( core_project["releases"][core_version], CORE_PACKAGE, core_version ) @@ -371,7 +384,12 @@ def candidate_action( now: Optional[str] = None, owner_is_failed: Optional[Callable[[str], bool]] = None, ) -> tuple[Action, str]: - """Whether to qualify ``pair`` now, and why.""" + """Whether to qualify ``pair`` now, and why. + + A promotable pair that advances main but was qualified against an + earlier certificate is requalified once per main commit, so automatic + promotion never waits on a manual ``--force``. + """ current = certified_order_key(bundle) record = bundle["data_releases"]["uk"] @@ -402,13 +420,123 @@ def candidate_action( ) ): return "qualify", "recoverable abandoned staged attempt" + stale = ( + receipt is not None + and receipt.promotable + and dominates(pair.order_key, current) + and stale_qualification(receipt, bundle, pair_dir, base_commit) + ) + if stale: + # Promotion fast-forwards main to the qualified commit, so it needs a + # qualification built on current main against main's certificate. + # Requalify once per main commit, without force, and one at a time. + if attempt is not None: + return "duplicate", ( + f"a requalification from base {base_commit[:12]} exists" + ) + if attempt_in_flight(pair_dir, now=now, owner_is_failed=owner_is_failed): + return "duplicate", "a requalification is already in flight" + return "qualify", f"promotable but {stale}; requalifying against current main" if receipt is not None and receipt.verdict != "incomplete": return "duplicate", f"pair already {receipt.verdict} in the ledger" if attempt is not None: return "duplicate", f"an attempt from base {base_commit[:12]} exists" + if attempt_in_flight(pair_dir, now=now, owner_is_failed=owner_is_failed): + # Runs resolved on different main commits must not both stage. + return "duplicate", "an attempt for this pair is in flight" return "qualify", "new pair, or an incomplete pair on a new base commit" +def qualified_base(pair_dir: Path) -> Optional[str]: + """The main commit prefix the pair's published receipt was built on. + + An attempt's key is its base commit's first 12 characters, with ``-`` + for a repeat on the same base. Without ``latest_attempt.json`` (pairs + published before it existed), the attempt whose own receipt is the + published one names it. ``None`` when no attempt does. + """ + + pointer = pair_dir / "latest_attempt.json" + if pointer.exists(): + return str(read_json(pointer)["attempt_key"]).split("-")[0] + published = pair_dir / "qualification_receipt.json" + if not published.exists(): + return None + digest = QualificationReceipt.model_validate(read_json(published)).unreviewed_sha256 + matching = [ + path.parent.name + for path in sorted(pair_dir.glob("attempts/*/qualification_receipt.json")) + if QualificationReceipt.model_validate(read_json(path)).unreviewed_sha256 + == digest + ] + return matching[-1].split("-")[0] if matching else None + + +def stale_qualification( + receipt: QualificationReceipt, + bundle: Mapping[str, Any], + pair_dir: Path, + main_commit: str, +) -> Optional[str]: + """Why a promotable receipt cannot be promoted onto current main, if so. + + Promotion fast-forwards main to the qualified commit, which needs the + qualification to be built on current main and compared against main's + certificate. + """ + + if receipt.previous_certificate != previous_certificate(bundle): + return "qualified against a superseded baseline" + base = qualified_base(pair_dir) + if base is None: + return "qualified on an unrecorded main commit" + if base != main_commit[:12]: + return f"qualified on main {base}, not the current {main_commit[:12]}" + return None + + +def attempt_in_flight( + pair_dir: Path, + *, + now: Optional[str], + owner_is_failed: Optional[Callable[[str], bool]], +) -> bool: + """A staged attempt newer than the published one that is still running: + it has no receipt yet and is not abandoned (a sent replay request is + still running).""" + + stages = list((pair_dir / "attempts").glob("*/staging.json")) + if not stages: + return False + newest = max(stages, key=lambda path: int(read_json(path)["sequence"])).parent + pointer = pair_dir / "latest_attempt.json" + published = read_json(pointer)["attempt_key"] if pointer.exists() else None + return ( + newest.name != published + and not (newest / "qualification_receipt.json").exists() + and not attempt_is_recoverable(newest, now=now, owner_is_failed=owner_is_failed) + ) + + +def requalification_target( + receipts: Iterable[QualificationReceipt], + bundle: Mapping[str, Any], + ledger_root: Path, + main_commit: str, +) -> Optional[CandidatePair]: + """The promotion target, when it must be requalified on current main.""" + + target = select_promotion_target(list(receipts), certified_order_key(bundle)) + if target is None: + return None + pair_dir = pair_directory(ledger_root, "uk", target.pair_id) + return ( + target.pair + if stale_qualification(target, bundle, pair_dir, main_commit) + else None + ) + + # --- Preparing the candidate commit ----------------------------------------- #: Test files that pin the certified UK pair literally. Promotion regenerates @@ -1207,6 +1335,9 @@ class PromotionPlan: reason: str = "" #: The qualified commit the ready branch must still be at when promoted. ready_commit: Optional[str] = None + #: The target, when its qualification compared against an earlier + #: certificate than main's: it needs requalifying before promotion. + requalify: Optional[CandidatePair] = None def plan_promotion( @@ -1258,9 +1389,10 @@ def plan_promotion( close_branches=sorted(close), reason=( f"{target.pair.order_key} advances {current}, but its previous " - "certificate differs from current main; requalify with --force " + "certificate differs from current main; it is requalified " "against current main before promotion" ), + requalify=target.pair, ) branch = candidate_branch(target.pair) qualified_commit = target.harness.git_commit diff --git a/tests/fixtures/uk_replay/README.md b/tests/fixtures/uk_replay/README.md index 398abb5c9..3c15e2c82 100644 --- a/tests/fixtures/uk_replay/README.md +++ b/tests/fixtures/uk_replay/README.md @@ -8,3 +8,10 @@ replay interface. The scorecard computes digests with `rfc8785==0.1.4`; Refresh by copying the files from a newer scorecard commit and updating the commit above; never edit them by hand. + +`receipt_6.4.1_attempt1/` holds the first real replay request and receipt, for +the pair policyengine 6.4.1, policyengine-uk 2.130.1 and uk-data 1.58.1. They +were copied unchanged from the scorecard's `replay-state` branch at commit +`4166508f` (`requests//`). The receipt holds aggregate fiscal +totals only. `tests/test_qualification_contracts.py` checks that the replay +policy fails it on exactly what the d1344 decision says it should. diff --git a/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REPLAY_RECEIPT.json b/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REPLAY_RECEIPT.json new file mode 100644 index 000000000..b265bfb54 --- /dev/null +++ b/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REPLAY_RECEIPT.json @@ -0,0 +1,4480 @@ +{ + "corpus": { + "corpus_version": "uk-fiscal-event-replay/1", + "events": { + "autumn_budget_2024": { + "artifacts": 30, + "comparison_provenance_sha256": "892ef8cf5e66c21d91d96a344236fb51071b7e43d885bb489c3eed0bdeb5f390", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "registry_sha256": "89784d34920fb2d6aae702fb4028f84d7bf87f1694df77a0c9c5d255edbcdd34", + "run_manifest_sha256": "593a3e37a69faf3c5c4afeda26375aba2e748c208b8193178f6fbef179da6761", + "staging_manifest_sha256": "e9abe69562c9f8c0ccc2973331acb66d5b7d2971be7a22d257fb3a64057f8afb", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_years": [] + }, + "autumn_statement_2023": { + "artifacts": 20, + "comparison_provenance_sha256": "f3b7f083abea6eaeb584964b5e677ffc49ea1b2ad2e96a023453a25f7f0e6f3e", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "registry_sha256": "197d5583f0171104c673ca1f40b402a77ca2b003f496de37d6bc913fb9fd1021", + "run_manifest_sha256": "debb2ffec8011fe577e57ef3e334f72aa0e50d90ab46a8536ab2c231c2259c31", + "staging_manifest_sha256": "f2ed6a429597e7bc42e46398cdb03f69d92511c266762c4bfde6d00cf4beae08", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2023": { + "artifacts": 8, + "comparison_provenance_sha256": "c2beddb0bafb0c7a92ec20ed8a061acd1012f4cced9c3776b0abcb95bdd0ba4d", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "registry_sha256": "0a78bfe32712bd30bdf9137bdf182e82a266c7da0e9885dd61cac6ef99a76357", + "run_manifest_sha256": "e242bf40a01f186fd6b7c759d8d3927058a5a9e382d08d48df618f28199a6d64", + "staging_manifest_sha256": "7a74c574a781e37e45d5adb05afc694b4ba3fb1665fb45f5630c063ac75bec51", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2024": { + "artifacts": 15, + "comparison_provenance_sha256": "58478003451a69686b08c5bc90d8937f9614c5a19b384e0ea735e59d9cd4d950", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "registry_sha256": "64a3bccfd950e868ffc5d6261bf3903df88a3ebedc9b4444494acc717259817a", + "run_manifest_sha256": "0bce7f6e84de2cec8305ecfc8e769cd2c79e73ec736feb8282e9adaf944a4592", + "staging_manifest_sha256": "161ae8d04e51c4180d1c78ccd5fd21d0fae410f4d9237228bd7c2d9e61b90d37", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_statement_2025": { + "artifacts": 12, + "comparison_provenance_sha256": "93a00d8930bc4cac1fbdb6986fd9bdc718fff5d0fd302df2aa07488fa8f10327", + "full_supported_grid_complete": false, + "inert_measure_years": [ + { + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + } + ], + "registry_sha256": "d01ce0612f3f24830a8d8ea0abaaa6b2626963316b16750310d303f404fbab5b", + "run_manifest_sha256": "da4c040fee52a6338eba98d0b9d9f8ad88884df8ec132b4264c74c66a5a3ae50", + "staging_manifest_sha256": "d55ee087c5922f63badb8d55c888f7f6a469b6185950cdb585a5ae7eebb7b20c", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_years": [] + } + } + }, + "diff": { + "base_variant_id": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "diff_sha256": "b05a9a5fa31ee15fa34a33dcf52502d11fcd716a5892388124ea3504f0c5b180", + "flag_counts": { + "construction": 6, + "coverage": 1, + "inertness": 8, + "mapping": 0, + "numerical": 312, + "structural": 25, + "support": 34 + }, + "flagged_rows": [ + { + "base_gbp": null, + "event": "autumn_budget_2024", + "flags": [ + "coverage" + ], + "head_gbp": 574999000.021347, + "key": "obr_pmd_nov2025:11682:73295721bdfdb9e8", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 0.0, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "construction" + ], + "head_gbp": 574999000.021347, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 0.0, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 574999000.021347, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 4655334129.036697, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2321626476.090164, + "key": "obr_pmd_nov2025:11683:b9096ecd110093bf", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 4655334129.036697, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 2321626476.090164, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 4655334129.036697, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2321626476.090164, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 4793563953.670055, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2397857290.327587, + "key": "obr_pmd_nov2025:11684:0d39628e9b763f67", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 4793563953.670055, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 2397857290.327587, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 4793563953.670055, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2397857290.327587, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 4952429950.859825, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2484671863.66201, + "key": "obr_pmd_nov2025:11685:440412b0fb8c6e2d", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 4952429950.859825, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 2484671863.66201, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 4952429950.859825, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2484671863.66201, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 5112968146.86298, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2572403121.83226, + "key": "obr_pmd_nov2025:11686:9ceb140875d859b8", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 5112968146.86298, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 2572403121.83226, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 5112968146.86298, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2572403121.83226, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 5273812739.539009, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2660288740.9552917, + "key": "obr_pmd_nov2025:11687:f738e039412d9872", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": 5273812739.539009, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "construction" + ], + "head_gbp": 2660288740.9552917, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": 5273812739.539009, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2660288740.9552917, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": -8163494973.898804, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7061749331.279724, + "key": "obr_pmd_nov2025:11665:72598886f546ef8c", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2025 + }, + { + "base_gbp": 24174758043.215755, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 23874647882.154022, + "key": "obr_pmd_nov2025:11671:291070a94715062c", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2025 + }, + { + "base_gbp": 16011263069.316952, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 16812898550.874298, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2025 + }, + { + "base_gbp": -8163494973.898804, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7061749331.279724, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2025 + }, + { + "base_gbp": -1251612208.7499237, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1429307494.3798141, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2025 + }, + { + "base_gbp": -8507737534.679443, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7376939848.9661255, + "key": "obr_pmd_nov2025:11666:e4413c7487d85ddf", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2026 + }, + { + "base_gbp": 24754424875.897583, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 24413340740.52112, + "key": "obr_pmd_nov2025:11672:0b55202125025854", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2026 + }, + { + "base_gbp": 16246687341.218138, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 17036400891.554995, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2026 + }, + { + "base_gbp": -8507737534.679443, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7376939848.9661255, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2026 + }, + { + "base_gbp": -1260499988.3872528, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1431638070.3618393, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2026 + }, + { + "base_gbp": -8783977312.67981, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7564046836.411194, + "key": "obr_pmd_nov2025:11667:35345df3785dfdea", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2027 + }, + { + "base_gbp": 25206449240.09022, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 24832078611.256916, + "key": "obr_pmd_nov2025:11673:94b8906a8929f113", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2027 + }, + { + "base_gbp": 16422471927.410412, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 17268031774.845722, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2027 + }, + { + "base_gbp": -8783977312.67981, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7564046836.411194, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2027 + }, + { + "base_gbp": -1264763359.1383667, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1433904842.3321228, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2027 + }, + { + "base_gbp": -8988426278.909851, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7898227103.010376, + "key": "obr_pmd_nov2025:11668:a076a7d6b3992e7c", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": 25618300125.581856, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 25217865979.05774, + "key": "obr_pmd_nov2025:11674:6dd4d473c559cbea", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": 16629873846.672005, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 17319638876.047363, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": -8988426278.909851, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7898227103.010376, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": -1282042489.6673584, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1446313392.5655975, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": -9404061793.93042, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -8092876935.234375, + "key": "obr_pmd_nov2025:11669:1cedd72c387ad84b", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2029 + }, + { + "base_gbp": 26148779059.003807, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 25792458746.42136, + "key": "obr_pmd_nov2025:11675:286359ea5a53fc94", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2029 + }, + { + "base_gbp": 16744717265.073387, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 17699581811.186985, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2029 + }, + { + "base_gbp": -9404061793.93042, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -8092876935.234375, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2029 + }, + { + "base_gbp": -1298741997.8063965, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1450619444.7683945, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2029 + }, + { + "base_gbp": 1258445339.7229428, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1940272255.14898, + "key": "obr_pmd_nov2025:11617:414a87b10621e433", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1258445339.7229428, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1940272255.14898, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1258445339.7229428, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1940272255.14898, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1292278978.386702, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1966545651.307916, + "key": "obr_pmd_nov2025:11618:ad4256274cc146d3", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 1292278978.386702, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1966545651.307916, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 1292278978.386702, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1966545651.307916, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 1322999530.8546522, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2044395945.9878988, + "key": "obr_pmd_nov2025:11619:8726edda9aa7b020", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 1322999530.8546522, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2044395945.9878988, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 1322999530.8546522, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2044395945.9878988, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 1354856060.444217, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2138101665.0296612, + "key": "obr_pmd_nov2025:11620:5446e6157f4ef4ec", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 1354856060.444217, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2138101665.0296612, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 1354856060.444217, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2138101665.0296612, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 1388030829.5242352, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2163020750.92759, + "key": "obr_pmd_nov2025:11621:8bf103a18fecf781", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 1388030829.5242352, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2163020750.92759, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 1388030829.5242352, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2163020750.92759, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 2039004728.2579594, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2182277423.2901382, + "key": "obr_pmd_nov2025:11647:b269082735adb83b", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2039004728.2579594, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2182277423.2901382, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2039004728.2579594, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2182277423.2901382, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2106518390.8636017, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2254534269.309538, + "key": "obr_pmd_nov2025:11648:48bd77ddd06dec27", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2106518390.8636017, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2254534269.309538, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2106518390.8636017, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2254534269.309538, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2182605140.493597, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2335967764.6855392, + "key": "obr_pmd_nov2025:11649:910077d14a180f90", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2182605140.493597, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2335967764.6855392, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2182605140.493597, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2335967764.6855392, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2259267108.0862694, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2418086716.0288925, + "key": "obr_pmd_nov2025:11650:068acce8b0873200", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2259267108.0862694, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2418086716.0288925, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2259267108.0862694, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2418086716.0288925, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2336375935.3248425, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2500617321.3659096, + "key": "obr_pmd_nov2025:11651:9a7327566a66890c", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 2336375935.3248425, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2500617321.3659096, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 2336375935.3248425, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2500617321.3659096, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 1713880895.879209, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1477704209.4028206, + "key": "obr_pmd_nov2025:22853:636080d8aa33dd8e", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2024 + }, + { + "base_gbp": 1713880895.879209, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1477704209.4028206, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2024 + }, + { + "base_gbp": 1713880895.879209, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1477704209.4028206, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2024 + }, + { + "base_gbp": 349882534.4296024, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 175460925.4459455, + "key": "obr_pmd_nov2025:22854:edbde987f66af165", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 349882534.4296024, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 175460925.4459455, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 349882534.4296024, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 175460925.4459455, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 367920017.09756255, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 184684987.91397357, + "key": "obr_pmd_nov2025:22855:dbc5baf2a54734dc", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 367920017.09756255, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 184684987.91397357, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 367920017.09756255, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 184684987.91397357, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 397324703.9610295, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 216206005.17124224, + "key": "obr_pmd_nov2025:22856:c29981419df6c6f6", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 397324703.9610295, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 216206005.17124224, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 397324703.9610295, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 216206005.17124224, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 417582943.08061886, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 217874757.0607648, + "key": "obr_pmd_nov2025:22857:92bf18d3bd843235", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 417582943.08061886, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 217874757.0607648, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 417582943.08061886, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 217874757.0607648, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 453160743.2087734, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 228829386.94062686, + "key": "obr_pmd_nov2025:22858:6050e74df71edc1c", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": 453160743.2087734, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 228829386.94062686, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": 453160743.2087734, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 228829386.94062686, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10542:89ab4668e579ad56", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10548:f2f563200611f223", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10560:3a6d16cff489f44c", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:21461:f9e31391303b473d", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "base_gbp": -11082510181.993792, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -10630356167.505722, + "key": "obr_pmd_nov2025:10549:7322b894d39fa3ec", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": 86201088.67014313, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 241090644.10290527, + "key": "obr_pmd_nov2025:21462:c775903b43caf723", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": -10996309093.323648, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -10389265523.402817, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": -11086458169.228767, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -10632796331.321083, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": 3947987.234975338, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 2440163.8153615, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": 86201088.67014313, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 241090644.10290527, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": -11685772883.678026, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11329957617.157879, + "key": "obr_pmd_nov2025:10550:68081ae97974ee1e", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": 94249253.54277039, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 266181343.4298706, + "key": "obr_pmd_nov2025:21463:fd9bc11d19465ecb", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": -11591523630.135256, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11063776273.728008, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": -11689444668.923279, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11332885657.203163, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": 94249253.54277039, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 266181343.4298706, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": -12070073648.808764, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11791691029.942415, + "key": "obr_pmd_nov2025:10551:67fe35e4947b995a", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": 104678630.74134064, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 282034438.7785034, + "key": "obr_pmd_nov2025:21464:190052623cdf04c0", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": -11965395018.067423, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11509656591.163912, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": -12073479284.911896, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11795774735.615051, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": 104678630.74134064, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 282034438.7785034, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": -12356159334.317142, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -12126516751.307087, + "key": "obr_pmd_nov2025:10552:16a41c96f88267ae", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": 109333370.59695435, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 292445236.05249023, + "key": "obr_pmd_nov2025:21465:6ea19652f3e3ea6b", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": -12246825963.720188, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -11834071515.254597, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": -12359540300.262665, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -12132335398.171082, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": 3380965.9455218315, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 5818646.863994598, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": 109333370.59695435, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 292445236.05249023, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": -12663862924.5513, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -12442433002.339989, + "key": "obr_pmd_nov2025:10553:39c2cf71f48d4440", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": 109787606.17053986, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 303440070.03996277, + "key": "obr_pmd_nov2025:21466:43674024394e1bc0", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": -12554075318.38076, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -12138992932.300026, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": -12667318767.890686, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -12449876132.117287, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": 3455843.3393855095, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 7443129.777297974, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": 109787606.17053986, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 303440070.03996277, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10590:9fb656d369cf6c02", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10596:482eac698ab9c181", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10608:8d76207d856fc00c", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:21497:197a81ed0459bc7d", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "base_gbp": -512170359.34750974, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -444404910.7621303, + "key": "obr_pmd_nov2025:10597:37b420855b7005a0", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": 9706978.086303711, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 39312296.15826416, + "key": "obr_pmd_nov2025:21498:3fa4108a7eac0a15", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": -502463381.26120603, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -405092614.60386616, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": -512118439.03477466, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -444404910.7621303, + "key": "ni_class_2", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": -51920.31273508072, + "event": "autumn_statement_2023", + "flags": [ + "structural" + ], + "head_gbp": 0.0, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": 9706978.086303711, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 39312296.15826416, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2024 + }, + { + "base_gbp": -519186159.4330233, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -459264249.43539166, + "key": "obr_pmd_nov2025:10598:28f9366b82391330", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": 9353628.809425354, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 39489133.344177246, + "key": "obr_pmd_nov2025:21499:909bf2ab3554e230", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": -509832530.6235979, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -419775116.0912144, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": -519141267.9473098, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -459264249.43539166, + "key": "ni_class_2", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": -44891.4857134819, + "event": "autumn_statement_2023", + "flags": [ + "structural" + ], + "head_gbp": 0.0, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": 9353628.809425354, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 39489133.344177246, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2025 + }, + { + "base_gbp": -542530999.0906498, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -454992560.1827409, + "key": "obr_pmd_nov2025:10599:0cdd3c6417e28a14", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": 11441880.627342224, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 37503348.28427124, + "key": "obr_pmd_nov2025:21500:3cd35347ee5fc266", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": -531089118.4633076, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -417489211.8984697, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": -542485937.01707, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -454992560.1827409, + "key": "ni_class_2", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": -45062.07357978821, + "event": "autumn_statement_2023", + "flags": [ + "structural" + ], + "head_gbp": 0.0, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": 11441880.627342224, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 37503348.28427124, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2026 + }, + { + "base_gbp": -1338847352.1526794, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -474097034.6309568, + "key": "obr_pmd_nov2025:10600:d0828a5bfb537ca4", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": 11933081.293174744, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 41829053.17868042, + "key": "obr_pmd_nov2025:21501:53e1c92d4a4f4e96", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": -1326914270.8595047, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -432267981.4522764, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": -552039153.5443193, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -474097034.6309568, + "key": "ni_class_2", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": -786808198.6083603, + "event": "autumn_statement_2023", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": 11933081.293174744, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 41829053.17868042, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2027 + }, + { + "base_gbp": -1054555794.4022624, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -491167651.8830828, + "key": "obr_pmd_nov2025:10601:756ded1466aa84b3", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": 11951206.26462555, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 41993236.97143555, + "key": "obr_pmd_nov2025:21502:1ea7fa7e0cfb40da", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": -1042604588.1376369, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -449174414.91164726, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": -558680740.925205, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -491167651.8830828, + "key": "ni_class_2", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": -495875053.47705746, + "event": "autumn_statement_2023", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": 11951206.26462555, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 41993236.97143555, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10566:0e8d21e52f88eb57", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10572:2ff1949c7ce7acb3", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10584:443a5169e8191f85", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:21479:4eea71bff070729b", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "base_gbp": -557157541.6621337, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -324495750.0986695, + "key": "obr_pmd_nov2025:10573:22a57b73322dcf42", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2024 + }, + { + "base_gbp": 6461029.071586609, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 18655108.79447937, + "key": "obr_pmd_nov2025:21480:4e24d4d5f036370a", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2024 + }, + { + "base_gbp": -550696512.5905471, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -305840641.30419016, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2024 + }, + { + "base_gbp": -557157541.6621337, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -324495750.0986695, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2024 + }, + { + "base_gbp": 6461029.071586609, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 18655108.79447937, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2024 + }, + { + "base_gbp": -564695883.8826604, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -330653144.5776048, + "key": "obr_pmd_nov2025:10574:46d6b72964d38975", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2025 + }, + { + "base_gbp": 6347881.413131714, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 20918336.49822998, + "key": "obr_pmd_nov2025:21481:2b2973b0ac23d6e9", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2025 + }, + { + "base_gbp": -558348002.4695287, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -309734808.0793748, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2025 + }, + { + "base_gbp": -564695883.8826604, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -330653144.5776048, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2025 + }, + { + "base_gbp": 6347881.413131714, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 20918336.49822998, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2025 + }, + { + "base_gbp": -583538584.2492275, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -348326831.56943035, + "key": "obr_pmd_nov2025:10575:e617707f9625fbba", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2026 + }, + { + "base_gbp": 7701561.54548645, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 22048259.627441406, + "key": "obr_pmd_nov2025:21482:753c71d52bde5307", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2026 + }, + { + "base_gbp": -575837022.7037411, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -326278571.94198895, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2026 + }, + { + "base_gbp": -583538584.2492275, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -348326831.56943035, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2026 + }, + { + "base_gbp": 7701561.54548645, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 22048259.627441406, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2026 + }, + { + "base_gbp": -607032565.8024693, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -369477043.1628766, + "key": "obr_pmd_nov2025:10576:033898ebb4865f78", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2027 + }, + { + "base_gbp": 8119534.424377441, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 23584034.71257019, + "key": "obr_pmd_nov2025:21483:8913a74a278eaeb2", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2027 + }, + { + "base_gbp": -598913031.3780918, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -345893008.4503064, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2027 + }, + { + "base_gbp": -607032565.8024693, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -369477043.1628766, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2027 + }, + { + "base_gbp": 8119534.424377441, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 23584034.71257019, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2027 + }, + { + "base_gbp": -631251667.9559898, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -390706828.95479107, + "key": "obr_pmd_nov2025:10577:55cd081c955ca847", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2028 + }, + { + "base_gbp": 8283750.790786743, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 24746395.838638306, + "key": "obr_pmd_nov2025:21484:a26658ab9b55efac", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2028 + }, + { + "base_gbp": -622967917.1652031, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -365960433.11615276, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2028 + }, + { + "base_gbp": -631251667.9559898, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -390706828.95479107, + "key": "ni_class_4", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2028 + }, + { + "base_gbp": 8283750.790786743, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 24746395.838638306, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:21629:a71ef831d6e1cb87", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + }, + { + "base_gbp": null, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:21635:7e6b2c17ec9be5e4", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "autumn_statement_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + }, + { + "base_gbp": -584567128.8834467, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1687110967.884717, + "key": "obr_pmd_nov2025:21630:bf9643f47e057983", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -584567128.8834467, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1687110967.884717, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -35532741.909638405, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -132222067.57011127, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -549034386.9738083, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1554888900.3146057, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -799767145.6307344, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1883075660.240798, + "key": "obr_pmd_nov2025:21631:75b9415771e2e20c", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -799767145.6307344, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1883075660.240798, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -74558905.50565815, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -145070938.0732565, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -725208240.1250763, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1738004722.1675415, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -833673303.0592041, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1917164100.9136887, + "key": "obr_pmd_nov2025:21632:353c5d3f922bfc89", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -833673303.0592041, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1917164100.9136887, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -79044897.4950409, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -149931822.89990997, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -754628405.5641632, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1767232278.0137787, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -859073219.5275888, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1979148274.9173164, + "key": "obr_pmd_nov2025:21633:5ce95c1559b8bc39", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -859073219.5275888, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1979148274.9173164, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -83591867.4580164, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -163561592.63858414, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -775481352.0695724, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1815586682.2787323, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -859833233.0249033, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2000162838.6384335, + "key": "obr_pmd_nov2025:21634:ff94f5b54b89d1ee", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -859833233.0249033, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2000162838.6384335, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -86662248.82321262, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -169179610.90740013, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -773170984.2016907, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1830983227.7310333, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -3693209247.487488, + "event": "spring_budget_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10231:ce0a1472426c8430", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10237:2f261725299509e1", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "base_gbp": -3693209247.487488, + "event": "spring_budget_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "base_gbp": -4297546474.76123, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3004426501.3733215, + "key": "obr_pmd_nov2025:10232:aa24b205227c35bd", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -4297546474.76123, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3004426501.3733215, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -4297546474.76123, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3004426501.3733215, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -5395125608.308472, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3300744216.6347656, + "key": "obr_pmd_nov2025:10233:f9cad5be5ec138c8", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2025 + }, + { + "base_gbp": -5395125608.308472, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3300744216.6347656, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2025 + }, + { + "base_gbp": -5395125608.308472, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3300744216.6347656, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2025 + }, + { + "base_gbp": -6040729276.933105, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3540262930.7841797, + "key": "obr_pmd_nov2025:10234:7fb22cdd33e49932", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -6040729276.933105, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3540262930.7841797, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -6040729276.933105, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3540262930.7841797, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -6456452398.171143, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3725202663.6395264, + "key": "obr_pmd_nov2025:10235:fbb8ccda250b78e9", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -6456452398.171143, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3725202663.6395264, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -6456452398.171143, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3725202663.6395264, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -2321120.9429092407, + "event": "spring_budget_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:20832:882117c50af13444", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + }, + { + "base_gbp": -2321120.9429092407, + "event": "spring_budget_2023", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + }, + { + "base_gbp": -1793717.8464126587, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -239031104.5414734, + "key": "obr_pmd_nov2025:20833:270c17d92e54dca4", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -1793717.8464126587, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -239031104.5414734, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -1793717.8464126587, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -239031104.5414734, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -1723210.1202392578, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -251689860.3839264, + "key": "obr_pmd_nov2025:20834:e0b34fcdbf83f61a", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -1723210.1202392578, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -251689860.3839264, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -1723210.1202392578, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -251689860.3839264, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -1925531.0124588013, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -255168633.24157715, + "key": "obr_pmd_nov2025:20835:1e730fccbdf4c015", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -1925531.0124588013, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -255168633.24157715, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -1925531.0124588013, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -255168633.24157715, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -2069178.6504898071, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -266037263.24046326, + "key": "obr_pmd_nov2025:20836:76beca3818ed1fe1", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": -2069178.6504898071, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -266037263.24046326, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": -2069178.6504898071, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -266037263.24046326, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10950:cfbd201bdcc71ff6", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10956:fe78e076cc7bb65b", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:22169:cfddc3a6fdeecca3", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": -11082510181.993792, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -10630356167.505722, + "key": "obr_pmd_nov2025:10957:34718787787f7089", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 86201088.67014313, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 241090644.10290527, + "key": "obr_pmd_nov2025:22170:45627d3976bd492a", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -10996309093.323648, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -10389265523.402817, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -11086458169.228767, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -10632796331.321083, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 3947987.234975338, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2440163.8153615, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 86201088.67014313, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 241090644.10290527, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -11685772883.678026, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11329957617.157879, + "key": "obr_pmd_nov2025:10958:c7a912de7a1a9114", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 94249253.54277039, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 266181343.4298706, + "key": "obr_pmd_nov2025:22171:93d8943dfe121528", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -11591523630.135256, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11063776273.728008, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -11689444668.923279, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11332885657.203163, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 94249253.54277039, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 266181343.4298706, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -12070073648.808764, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11791691029.942415, + "key": "obr_pmd_nov2025:10959:424db47edb88e339", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 104678630.74134064, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 282034438.7785034, + "key": "obr_pmd_nov2025:22172:8c021f0cf80fe71b", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -11965395018.067423, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11509656591.163912, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -12073479284.911896, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11795774735.615051, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 104678630.74134064, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 282034438.7785034, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -12356159334.317142, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -12126516751.307087, + "key": "obr_pmd_nov2025:10960:fa02958b7f130d94", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 109333370.59695435, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 292445236.05249023, + "key": "obr_pmd_nov2025:22173:3beb0f8e7601b1e2", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -12246825963.720188, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -11834071515.254597, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -12359540300.262665, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -12132335398.171082, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 3380965.9455218315, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 5818646.863994598, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 109333370.59695435, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 292445236.05249023, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -12663862924.5513, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -12442433002.339989, + "key": "obr_pmd_nov2025:10961:cc058843879051af", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 109787606.17053986, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 303440070.03996277, + "key": "obr_pmd_nov2025:22174:b9088212717e5ab4", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": -12554075318.38076, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -12138992932.300026, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": -12667318767.890686, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -12449876132.117287, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 3455843.3393855095, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 7443129.777297974, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 109787606.17053986, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 303440070.03996277, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10968:f0c2d5c4f10ba12c", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10974:fdd0e608076292a8", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:22187:cc8ffca6f2f5e590", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "base_gbp": -1114763387.7151809, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -649587042.2363234, + "key": "obr_pmd_nov2025:10975:497e18b9ba827fa3", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 12922100.778694153, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 37366356.46134949, + "key": "obr_pmd_nov2025:22188:9f98363a90ba075c", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -1101841286.9364867, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -612220685.7749739, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -1114763387.7151809, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -649587042.2363234, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 12922100.778694153, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 37366356.46134949, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": -1129913657.7188382, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -661746627.0762091, + "key": "obr_pmd_nov2025:10976:230199453880fc45", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 12695757.11100769, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 41882343.67687988, + "key": "obr_pmd_nov2025:22189:215059939a4b5efc", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -1117217900.6078305, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -619864283.3993292, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -1129913657.7188382, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -661746627.0762091, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 12695757.11100769, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 41882343.67687988, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": -1167913753.6299438, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -696073741.2305636, + "key": "obr_pmd_nov2025:10977:57eced9fbf6bab3b", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 15403093.633636475, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 44096154.611190796, + "key": "obr_pmd_nov2025:22190:dc5ace295177c84b", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -1152510659.9963074, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -651977586.6193728, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -1167913753.6299438, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -696073741.2305636, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 15403093.633636475, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 44096154.611190796, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": -1215475927.791809, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -738860279.7822428, + "key": "obr_pmd_nov2025:10978:58862ededbda2b44", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 16239100.814880371, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 47827344.751449585, + "key": "obr_pmd_nov2025:22191:6f3ee686a895fd60", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -1199236826.9769287, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -691032935.0307932, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -1215475927.791809, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -738860279.7822428, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 16239100.814880371, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 47827344.751449585, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": -1263944357.8132381, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -782252598.0831976, + "key": "obr_pmd_nov2025:10979:3e84f55a7dc5bd63", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 16707183.741485596, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 49616282.76098633, + "key": "obr_pmd_nov2025:22192:79bd1761809b7468", + "level": "source_row", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": -1247237174.0717525, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -732636315.3222113, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": -1263944357.8132381, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -782252598.0831976, + "key": "ni_class_4", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 16707183.741485596, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 49616282.76098633, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:10986:8084652420f5c808", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:22205:3c227ad5b7739a89", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "support" + ], + "head_gbp": null, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + }, + { + "base_gbp": -1491806310.2611084, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -779631545.703949, + "key": "obr_pmd_nov2025:10987:0f254c98c34bc660", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -336165102.39004517, + "key": "obr_pmd_nov2025:22206:49a0caec037600d2", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -1491806310.2611084, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1115796648.0939941, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -336165102.39004517, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -1491806310.2611084, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -779631545.703949, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -1592760827.531311, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -980457483.7055664, + "key": "obr_pmd_nov2025:10988:6a33f7863750611c", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -282526333.77780914, + "key": "obr_pmd_nov2025:22207:01b31afdbe23dae6", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -1592760827.531311, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1262983817.4833755, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -282526333.77780914, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -1592760827.531311, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -980457483.7055664, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -1720629444.0709229, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1131591240.399292, + "key": "obr_pmd_nov2025:10989:43638ad22300c9c3", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -307810876.10124016, + "key": "obr_pmd_nov2025:22208:6c19f800730945b5", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1720629444.0709229, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1439402116.5005322, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -307810876.10124016, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1720629444.0709229, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1131591240.399292, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1844209824.508789, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1274651222.2009888, + "key": "obr_pmd_nov2025:10990:e86af1dbeb4da64e", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -323556121.7815895, + "key": "obr_pmd_nov2025:22209:71f606ed0f936afe", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1844209824.508789, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1598207343.9825783, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -323556121.7815895, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1844209824.508789, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1274651222.2009888, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1972030891.008606, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1394705720.206482, + "key": "obr_pmd_nov2025:10991:11a476b01436c0db", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": 0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -363073039.4392204, + "key": "obr_pmd_nov2025:22210:0ea61ade0bedb0f5", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -1972030891.008606, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1757778759.6457024, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -0.0, + "event": "spring_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -363073039.4392204, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -1972030891.008606, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1394705720.206482, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": 264112675.57543182, + "event": "spring_statement_2025", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:23317:1b983354102bb14d", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 264112675.57543182, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 264112675.57543182, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -0.0, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 338556341.93270874, + "event": "spring_statement_2025", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:23318:dcc529f309fec572", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 338556341.93270874, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 338556341.93270874, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -0.0, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 436340777.1042633, + "event": "spring_statement_2025", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:23319:c6ce8500db20415e", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 436340777.1042633, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 436340777.1042633, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -0.0, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 626601979.8689651, + "event": "spring_statement_2025", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:23320:4c41d84635ed82ad", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": 626601979.8689651, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": 626601979.8689651, + "event": "spring_statement_2025", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": -0.0, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": -403656800.85860443, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -723462767.5098114, + "key": "obr_pmd_nov2025:23335:7e9f9d66b5194ff6", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2026 + }, + { + "base_gbp": -403656800.85860443, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -723462767.5098114, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2026 + }, + { + "base_gbp": -403656800.85860443, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -723462767.5098114, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2026 + }, + { + "base_gbp": -409490512.0053253, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -732669693.5323181, + "key": "obr_pmd_nov2025:23336:5d8fea1a1f3c86d6", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -409490512.0053253, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -732669693.5323181, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -409490512.0053253, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -732669693.5323181, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -414257872.6100464, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -746613549.421051, + "key": "obr_pmd_nov2025:23337:31f0f1693e3bba3c", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -414257872.6100464, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -746613549.421051, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -414257872.6100464, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -746613549.421051, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -421859102.8292618, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -753234184.9562225, + "key": "obr_pmd_nov2025:23338:27e34c93a163b4f0", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + }, + { + "base_gbp": -421859102.8292618, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -753234184.9562225, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + }, + { + "base_gbp": -421859102.8292618, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -753234184.9562225, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + } + ] + }, + "execution": { + "dataset_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "packaged_bundle_manifest_sha256": "fc1abd76660e9b0bdfe6319a2ae5f221847750e087143e5ba6da859a9422e333", + "packages": { + "policyengine": { + "record_sha256": "292af3479d12a7ad10cc8c84377e81cf5362d75b8289603c0948968f53a24790", + "version": "6.4.1", + "wheel_sha256": "550c3a9ae9c74d409d2fa81b7090850c2e5115b57f91ef1814f3c5fddaca36f2" + }, + "policyengine-core": { + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine-uk": { + "record_sha256": "365cc5256f85b474bd445403462e17e468a1c2f5da6cc2c2b1cbf626ac20e243", + "version": "2.130.1", + "wheel_sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + } + }, + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python": "3.12.3", + "release_bundle": { + "bundle_id": "uk-6.4.1", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "certified_data_build_id": "policyengine-uk-data-1.58.1", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.1", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": "policyengine-uk", + "model_version": "2.130.1", + "policyengine_version": "6.4.1" + }, + "requirements_sha256": "d1636b4d91af2a1e1802f5e7551a7bf3023ac1124db7730d31d05925725024d9", + "runner": "github-actions Linux X64" + }, + "integrity": { + "failures": [], + "status": "incomplete" + }, + "receipt_sha256": "e3cc6521f0cf2273fc1e7970983421d14a668afc5d21f829a71775b15873f34f", + "rejection": null, + "request_id": "d3d242e761b9b5177ae77a2528e1cd3dab306cb73350e79648551dc99c288412", + "results": { + "manifest_sha256": "9def79c1b2903ddabdc8bf70e618d478a3c2950deb49f6ec31ff5e3025718056", + "path": "requests/d3d242e761b9b5177ae77a2528e1cd3dab306cb73350e79648551dc99c288412/results/", + "ref": "replay-state", + "repository": "PolicyEngine/policyengine-scorecard" + }, + "schema": "policyengine.uk-replay-receipt/v1", + "status": "complete", + "variant_id": "pe6.4.1_uk2.130.1_policyengine-uk-data-1.58.1", + "variant_identity_sha256": "fb1d19d6469a82dd447fdba3aff65daab22da9ed654a718a572da8d0da40d774" +} diff --git a/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REQUEST.json b/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REQUEST.json new file mode 100644 index 000000000..35e82fcd3 --- /dev/null +++ b/tests/fixtures/uk_replay/receipt_6.4.1_attempt1/REQUEST.json @@ -0,0 +1,59 @@ +{ + "attempt": 1, + "candidate": { + "data": { + "build_id": "policyengine-uk-data-1.58.1", + "dataset": "enhanced_frs_2024_25", + "path": "enhanced_frs_2024_25.h5", + "release_manifest_sha256": "545797bf231cd3681e64b677065d921304e9a32f1e46e757a940a412b4136c17", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "size_bytes": 128409762 + }, + "expected_release_bundle": { + "bundle_id": "uk-6.4.1", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "certified_data_build_id": "policyengine-uk-data-1.58.1", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.1", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": "policyengine-uk", + "model_version": "2.130.1", + "policyengine_version": "6.4.1" + }, + "policyengine": { + "packaged_bundle_manifest_sha256": "fc1abd76660e9b0bdfe6319a2ae5f221847750e087143e5ba6da859a9422e333", + "source": { + "pypi": { + "wheel_sha256": "550c3a9ae9c74d409d2fa81b7090850c2e5115b57f91ef1814f3c5fddaca36f2" + } + }, + "version": "6.4.1" + }, + "policyengine_core": { + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine_uk": { + "version": "2.130.1", + "wheel_sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + } + }, + "compare_to": "latest_certified", + "kind": "released_bundle", + "requested_by": { + "commit": "0ccad05d2eecb99acbfa125599a419b51d07f0e1", + "note": "Built by hand by the W4 session on 2026-10-15 for policyengine 6.4.1 (release commit 0ccad05d), with released_bundle_request from policyengine.py PR #574 at 769d5efa, because the push workflow's NotifyConsumers request is not on main yet. No workflow run produced it.", + "repository": "PolicyEngine/policyengine.py" + }, + "schema": "policyengine.uk-replay-qualification-request/v1" +} diff --git a/tests/qualification_fixtures.py b/tests/qualification_fixtures.py index 5f27153bf..bf74fc960 100644 --- a/tests/qualification_fixtures.py +++ b/tests/qualification_fixtures.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import shutil +import subprocess from pathlib import Path from policyengine.provenance.qualification import ( @@ -98,5 +100,115 @@ def make_receipt( ) +def jq(expression: str, data) -> str: + """Apply ``expression`` to ``data`` with the real jq, printing strings + raw as gh's ``--jq`` does, so fakes of gh evaluate the code's own jq + rather than guess at it (round 6 of #635: a fake that picked fields by + name let a wrong source path pass). GitHub's ubuntu runners ship jq; a + machine without it fails these tests rather than skipping them.""" + + binary = shutil.which("jq") + assert binary, "jq is required to evaluate the code's jq expressions" + return subprocess.run( + [binary, "-r", expression], + input=json.dumps(data), + capture_output=True, + text=True, + check=True, + ).stdout + + +def _field(argv, name): + """The value of gh's ``-f name=value`` / ``-F name=value`` in ``argv``.""" + + for flag, value in zip(argv, argv[1:]): + if flag in ("-f", "-F") and value.startswith(f"{name}="): + return value.split("=", 1)[1] + return None + + +def rest_pulls_output(argv, listing, owner="PolicyEngine") -> str: + """What `gh api -X GET repos//pulls -f head=: ...` + prints for ``listing`` (gh pr list-shaped items: number, headRefName, + headRefOid, baseRefName (main), state (OPEN, CLOSED or MERGED), isDraft, + isCrossRepository): GitHub keeps only pull requests from ``owner``'s + branch, then filters by base and state, and the call's --jq runs over + REST-shaped objects with the real jq.""" + + head = _field(argv, "head") + head_owner, _, branch = head.partition(":") + base = _field(argv, "base") + state = _field(argv, "state") or "open" + page = [] + for pr in listing: + pr = {"baseRefName": "main", "state": "OPEN", "isDraft": True, **pr} + pr_owner = "someone" if pr.get("isCrossRepository") else owner + rest_state = "open" if pr["state"] == "OPEN" else "closed" + if ( + pr_owner != head_owner + or pr["headRefName"] != branch + or (base is not None and pr["baseRefName"] != base) + or (state != "all" and rest_state != state) + ): + continue + page.append( + { + "number": pr["number"], + "state": rest_state, + "draft": pr["isDraft"], + "merged_at": "2026-10-16T00:00:00Z" + if pr["state"] == "MERGED" + else None, + "base": {"ref": pr["baseRefName"]}, + "head": { + "ref": pr["headRefName"], + "sha": pr.get("headRefOid"), + "repo": {"full_name": f"{pr_owner}/policyengine.py"}, + }, + } + ) + return jq(argv[argv.index("--jq") + 1], page) + + +#: Each kind of base change and the timeline event GitHub records for it. +BASE_CHANGE_EVENTS = { + "manual": ("BASE_REF_CHANGED_EVENT", "BaseRefChangedEvent"), + "automatic": ( + "AUTOMATIC_BASE_CHANGE_SUCCEEDED_EVENT", + "AutomaticBaseChangeSucceededEvent", + ), +} + + +def graphql_base_changes_output(argv, changed) -> str: + """What the base-change GraphQL query prints for the pull request it + names. ``changed`` maps numbers to the kind of base change they had + ("manual" or "automatic"; a set of numbers means manual). As in GitHub, + a pull request's event is among the nodes only when the query's + itemTypes ask for its type (round 8 of #635: an automatic retarget is a + separate event type that a query for BASE_REF_CHANGED_EVENT misses).""" + + query = _field(argv, "query") + types = set(query[query.index("itemTypes: [") + 12 :].split("]")[0].split(", ")) + kinds = changed if isinstance(changed, dict) else dict.fromkeys(changed, "manual") + number = int(_field(argv, "number")) + item_type, typename = BASE_CHANGE_EVENTS.get(kinds.get(number), (None, None)) + page = { + "data": { + "repository": { + "pullRequest": { + "timelineItems": { + "nodes": [{"__typename": typename}] + if item_type in types + else [], + "pageInfo": {"hasNextPage": False}, + } + } + } + } + } + return jq(argv[argv.index("--jq") + 1], page) + + def golden(name: str) -> dict: return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) diff --git a/tests/test_qualification_contracts.py b/tests/test_qualification_contracts.py index 642d67477..9b55e534a 100644 --- a/tests/test_qualification_contracts.py +++ b/tests/test_qualification_contracts.py @@ -12,7 +12,10 @@ - the replay request, status and receipt mirrors agree with the scorecard's vendored JSON Schemas and reproduce its golden digests (differential); - a replay receipt counts only when bound to this request and these - artifacts; blocking flags fail, review flags need review. + artifacts; +- the replay policy (d1344-replay-regressions-v1) fails exactly the + regressions it names, sends every other flagged row to review, and is + monotone: adding a flagged row never makes the verdict better. """ from __future__ import annotations @@ -28,6 +31,7 @@ from policyengine.provenance.canonical import canonical_sha256 from policyengine.provenance.qualification import ( + REPLAY_POLICY_ID, REQUIRED_GATES, GateResult, ReplayQualificationRequest, @@ -471,11 +475,19 @@ def _with_counts(sealed=True, **counts): return _sealed(doc) if sealed else doc -def test_golden_receipt_fails_on_its_structural_flag(): +def test_golden_receipt_needs_review_for_its_sign_reversal(): + """Its structural row moves £0.0000001 to -£123m: a sign reversal, which a + model fix can make legitimately, so a named reviewer decides.""" + doc = _with_requested_wheels(copy.deepcopy(golden("replay_receipt_golden"))) gate = _gate(_sealed(doc)) - assert gate.status == "fail" - assert "structural" in gate.summary + assert gate.status == "needs_review", gate.summary + assert gate.details["regressions"] == [] + assert sorted(item["level"] for item in gate.details["review_items"]) == [ + "head", + "measure_year", + "source_row", + ] @pytest.mark.parametrize( @@ -484,9 +496,12 @@ def test_golden_receipt_fails_on_its_structural_flag(): ({}, "pass"), ({"numerical": 2}, "needs_review"), ({"support": 1}, "needs_review"), - ({"numerical": 2, "coverage": 1}, "fail"), - ({"construction": 1}, "fail"), - ({"inertness": 1}, "fail"), + # The template row has values on both sides: not a lost result. + ({"numerical": 2, "coverage": 1}, "needs_review"), + ({"construction": 1}, "needs_review"), + # Not in the candidate's inert list: inert in the base only. + ({"inertness": 1}, "needs_review"), + ({"structural": 1}, "needs_review"), ({"mapping": 3}, "fail"), ], ) @@ -548,7 +563,10 @@ def test_blocking_flags_count_wherever_the_receipt_states_them(): assert "mapping" in _gate(_sealed(receipt)).summary -def test_an_inert_measure_year_without_a_flag_needs_review(): +def test_a_measure_year_the_candidate_lists_as_inert_fails(): + """No receipt yet records the base's inert measure-years, so no gap is + known, and an inert measure-year is a lost response.""" + receipt = _edit( _clean_receipt(), lambda r: r["corpus"]["events"][EVENT].update( @@ -556,7 +574,474 @@ def test_an_inert_measure_year_without_a_flag_needs_review(): ), ) gate = _gate(receipt) - assert gate.status == "needs_review" and "inert" in gate.summary + assert gate.status == "fail" and "lost_response 1" in gate.summary + assert gate.details["known_gaps"] == [] + + +MEASURE = "spring_budget_2023__uc_childcare_cap_increase" + + +def _receipt_with(rows=(), inert=(), complete=True, integrity="complete", counts=None): + """The golden receipt with these flagged rows (measure-year totals unless + a row says otherwise), counts that match them, and this event state.""" + + doc = _with_requested_wheels(copy.deepcopy(golden("replay_receipt_golden"))) + template = { + "event": EVENT, + "measure_key": MEASURE, + "year": 2026, + "level": "measure_year", + "key": "total", + } + doc["diff"]["flagged_rows"] = [{**template, **row} for row in rows] + tally = {key: 0 for key in doc["diff"]["flag_counts"]} + for row in doc["diff"]["flagged_rows"]: + for kind in row["flags"]: + tally[kind] += 1 + tally.update(counts or {}) + doc["diff"]["flag_counts"] = tally + event = doc["corpus"]["events"][EVENT] + event["inert_measure_years"] = [ + {"measure_key": measure, "year": year} for measure, year in inert + ] + event["full_supported_grid_complete"] = complete + doc["integrity"]["status"] = integrity + return _sealed(doc) + + +# The regression table of the d1344 decision record (second review), plus the +# bound's edges. Each: rows, candidate inert, grid complete, integrity, then +# the expected status and the kinds the gate must record. +POLICY_CASES = { + "sign reversal": ( + [{"flags": ["numerical", "structural"], "base_gbp": 1e8, "head_gbp": -1e8}], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "£0 against a material base, distinct worlds": ( + [{"flags": ["numerical", "structural"], "base_gbp": 1e8, "head_gbp": 0.0}], + (), + True, + "complete", + "fail", + {"lost_response"}, + ), + "exactly the bound to £0, no numerical flag": ( + [{"flags": ["structural"], "base_gbp": 1e6, "head_gbp": 0.0}], + (), + True, + "complete", + "fail", + {"lost_response"}, + ), + "negative material base to £0": ( + [{"flags": ["structural"], "base_gbp": -2.5e8, "head_gbp": 0.0}], + (), + True, + "complete", + "fail", + {"lost_response"}, + ), + "just below the bound to £0": ( + [{"flags": ["structural"], "base_gbp": 999_999.0, "head_gbp": 0.0}], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "£0 head (not a measure-year total)": ( + [ + { + "flags": ["structural"], + "level": "head", + "key": "universal_credit", + "base_gbp": 1e8, + "head_gbp": 0.0, + } + ], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "£0 source row against a material base (not a measure-year total)": ( + [ + { + "flags": ["structural"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": 1e8, + "head_gbp": 0.0, + } + ], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "numeric base result, missing candidate result": ( + [ + { + "flags": ["coverage"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": 5e8, + "head_gbp": None, + } + ], + (), + True, + "complete", + "fail", + {"lost_result"}, + ), + "coverage gain": ( + [ + { + "flags": ["coverage"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": None, + "head_gbp": 574_999_000.021347, + } + ], + (), + True, + "complete", + "needs_review", + {"coverage_gain"}, + ), + "null/null coverage, otherwise complete": ( + [ + { + "flags": ["coverage"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": None, + "head_gbp": None, + } + ], + (), + True, + "complete", + "needs_review", + {"coverage_without_values"}, + ), + "absent base, candidate inert and incomplete": ( + [ + { + "flags": ["coverage"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": None, + "head_gbp": None, + } + ], + ((MEASURE, 2026),), + False, + "incomplete", + "fail", + {"lost_response", "incomplete", "coverage_without_values"}, + ), + # As in the 6.4.1 receipt: a measure-year flag and a source-row flag + # whose candidate value is null, for a measure-year the candidate lists. + "candidate inert, flagged against the base": ( + [ + {"flags": ["inertness"], "base_gbp": 2.641127e8, "head_gbp": 0.0}, + { + "flags": ["inertness"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": 2.641127e8, + "head_gbp": None, + }, + ], + ((MEASURE, 2026),), + False, + "incomplete", + "fail", + {"lost_response", "incomplete"}, + ), + "inert in the base only": ( + [{"flags": ["inertness"], "base_gbp": 0.0, "head_gbp": 3e8}], + (), + True, + "complete", + "needs_review", + {"base_only_inertness"}, + ), + "same supported artifact missing on both sides": ( + [], + (), + False, + "incomplete", + "fail", + {"incomplete"}, + ), + "incomplete integrity with no incomplete event": ( + [], + (), + True, + "incomplete", + "fail", + {"incomplete"}, + ), + "candidate loses a supported year": ( + [ + { + "flags": ["support"], + "level": "source_row", + "key": "obr-row-1", + "base_gbp": 1e8, + "head_gbp": None, + } + ], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "construction": ( + [{"flags": ["construction"], "base_gbp": 1e8, "head_gbp": 1.2e8}], + (), + True, + "complete", + "needs_review", + {"flagged_move"}, + ), + "mapping": ( + [{"flags": ["mapping"], "base_gbp": 1e8, "head_gbp": 1e8}], + (), + True, + "complete", + "fail", + {"mapping"}, + ), +} + + +@pytest.mark.parametrize("name", sorted(POLICY_CASES)) +def test_the_replay_policy_decides_each_case_of_the_decision_record(name): + rows, inert, complete, integrity, status, kinds = POLICY_CASES[name] + receipt = _receipt_with(rows, inert, complete, integrity) + _validator("replay_receipt.v1.json").validate(receipt) + gate = _gate(receipt) + assert gate.status == status, (name, gate.summary) + recorded = { + item["kind"] + for item in gate.details["regressions"] + gate.details["review_items"] + } + assert recorded == kinds, (name, recorded) + + +def test_a_counted_flag_with_no_row_still_needs_review(): + gate = _gate(_receipt_with(counts={"numerical": 1})) + assert gate.status == "needs_review" + assert gate.details["review_items"] == [ + {"kind": "counted_flag", "flag": "numerical", "count": 1} + ] + + +def test_a_counted_mapping_flag_with_no_row_fails(): + """A blocking flag the counters state still fails, so no reviewer can + accept it.""" + + receipt = _receipt_with(counts={"mapping": 1}) + _validator("replay_receipt.v1.json").validate(receipt) + gate = _gate(receipt) + assert gate.status == "fail" + assert gate.details["regressions"] == [ + {"kind": "counted_flag", "flag": "mapping", "count": 1} + ] + + +def test_every_early_failure_carries_the_policy_record(): + import hashlib + from importlib.resources import files + + text = (files("policyengine.data.qualification") / "replay_policy.md").read_bytes() + clean = _receipt_with() + rejected = _rejected_receipt() + contradictory = _edit( + _clean_receipt(), + lambda r: r["corpus"]["events"][EVENT].update(artifacts=0), + ) + failed = copy.deepcopy(_with_requested_wheels(golden("replay_receipt_golden"))) + failed["status"] = "failed" + failed["integrity"] = { + "status": "failed", + "failures": [{"code": "construction_inert", "detail": "x"}], + } + unbound = copy.deepcopy(clean) + unbound["execution"]["dataset_sha256"] = "0" * 64 + cases = { + "rejected": _gate(rejected), + "contradiction": _gate(contradictory), + "failed": _gate(_sealed(failed)), + "unbound": _gate(_sealed(unbound)), + } + for kind, gate in cases.items(): + assert gate.status == "fail", kind + assert gate.details["policy"] == REPLAY_POLICY_ID + assert gate.details["policy_sha256"] == hashlib.sha256(text).hexdigest() + assert [r["kind"] for r in gate.details["regressions"]] == [kind], kind + assert gate.details["review_items"] == [] and gate.details["known_gaps"] == [] + + +def test_the_gate_records_the_policy_it_applied(): + import hashlib + from importlib.resources import files + + gate = _gate(_receipt_with()) + assert gate.status == "pass" + text = (files("policyengine.data.qualification") / "replay_policy.md").read_bytes() + assert gate.details["policy"] == REPLAY_POLICY_ID + assert gate.details["policy_sha256"] == hashlib.sha256(text).hexdigest() + assert REPLAY_POLICY_ID.encode() in text + assert gate.details["known_gaps"] == [] + + +def test_the_first_real_receipt_fails_on_its_inert_uc_measure_years(): + """policyengine 6.4.1 + policyengine-uk 2.130.1 + uk-data 1.58.1, receipt + e3cc6521…: scorecard's construction of spring_statement_2025's UC health + element change was inert in 2026-29, where the base computed £264.1m to + £626.6m. Those four lost responses and the incomplete grid fail it; its + other 346 flagged rows are review items.""" + + from collections import Counter + + folder = FIXTURES / "receipt_6.4.1_attempt1" + request = ReplayQualificationRequest.model_validate( + json.loads((folder / "REQUEST.json").read_text()) + ) + document = json.loads((folder / "REPLAY_RECEIPT.json").read_text()) + receipt = ReplayReceipt.from_document(document) + assert receipt.computed_receipt_sha256 == receipt.receipt_sha256 + gate = evaluate_replay_receipt( + request, _status(request.request_id, document), receipt + ) + assert gate.status == "fail" + lost = sorted( + (item["measure_key"], item["year"]) + for item in gate.details["regressions"] + if item["kind"] == "lost_response" + ) + measure = "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut" + assert lost == [(measure, year) for year in (2026, 2027, 2028, 2029)] + assert [ + item["event"] + for item in gate.details["regressions"] + if item["kind"] == "incomplete" + ] == ["spring_statement_2025"] + assert Counter(item["kind"] for item in gate.details["review_items"]) == { + "flagged_move": 345, + "coverage_gain": 1, + } + + +_ROW = st.fixed_dictionaries( + { + "flags": st.lists( + st.sampled_from( + [ + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction", + ] + ), + min_size=1, + max_size=3, + unique=True, + ), + "level": st.sampled_from(["source_row", "measure_year", "head"]), + "year": st.sampled_from([2025, 2026]), + "base_gbp": st.one_of( + st.none(), + st.just(0.0), + st.sampled_from([999_999.0, 1e6, -1e6, 2.5e8, -2.5e8]), + st.floats(-1e10, 1e10, allow_nan=False), + ), + "head_gbp": st.one_of( + st.none(), + st.just(0.0), + st.floats(-1e10, 1e10, allow_nan=False), + ), + } +) +_RANK = {"pass": 0, "needs_review": 1, "fail": 2} + + +@settings(max_examples=150, deadline=None) +@given( + rows=st.lists(_ROW, max_size=4), + extra=_ROW, + inert=st.lists(st.sampled_from([2025, 2026]), max_size=2, unique=True), +) +def test_the_replay_policy_is_monotone_and_never_passes_a_flag(rows, extra, inert): + """For every receipt: a flagged row never passes; mapping and lost results + always fail; fail is exactly "a regression was recorded"; and adding a + flagged row never makes the verdict better.""" + + listed = tuple((MEASURE, year) for year in inert) + complete = not listed + + def judge(these): + receipt = _receipt_with( + these, listed, complete, "complete" if complete else "incomplete" + ) + return _gate(receipt) + + def oracle(these): + """The decision record's outcome, restated independently: a + regression fails; any other flagged row needs review.""" + + for row in these: + base, head = row["base_gbp"], row["head_gbp"] + if ( + "mapping" in row["flags"] + or ("coverage" in row["flags"] and base is not None and head is None) + or ( + row["level"] == "measure_year" + and head == 0 + and base is not None + and abs(base) >= 1_000_000 + ) + ): + return "fail" + if listed: + return "fail" + return "needs_review" if these else "pass" + + before = judge(rows) + after = judge([*rows, extra]) + assert before.status == oracle(rows), (rows, before.summary) + assert after.status == oracle([*rows, extra]), (extra, after.summary) + assert _RANK[after.status] >= _RANK[before.status] + for gate, these in ((before, rows), (after, [*rows, extra])): + assert (gate.status == "fail") == bool(gate.details["regressions"]) + if these or listed: + assert gate.status != "pass" + if any( + "mapping" in row["flags"] + or ( + "coverage" in row["flags"] + and row["base_gbp"] is not None + and row["head_gbp"] is None + ) + for row in these + ): + assert gate.status == "fail" def test_the_comparison_base_must_be_the_one_requested(): diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index f210e1032..d2d9a81dc 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -52,8 +52,11 @@ VENDORED_SCHEMAS, gates, golden, + graphql_base_changes_output, + jq, make_pair, make_receipt, + rest_pulls_output, sha, ) @@ -127,6 +130,73 @@ def _resolver(**overrides): return resolve_candidate(**kwargs), manifest_bytes +def test_a_pinned_core_is_used_exactly_or_refused(): + """A requalification names its exact core: the certified one, if the + model accepts it, must not be chosen instead.""" + + resolved, _ = _resolver(model_version="2.126.8", core_version="3.34.1") + assert resolved.pair.core.version == "3.34.1" + assert _resolver(model_version="2.126.8")[0].pair.core.version == "3.33.0" + with pytest.raises(ValueError, match="not on PyPI"): + _resolver(model_version="2.126.8", core_version="9.9.9") + with pytest.raises(ValueError, match="requires policyengine-core"): + _resolver(model_version="2.126.6", core_version="3.33.0") + + +def test_resolve_refuses_a_pair_other_than_the_one_requested( + monkeypatch, tmp_path, capsys +): + """A requalification or recovery dispatch names one pair. If its versions + resolve to another (a moved data tag, another core), the run fails and + nothing else is qualified.""" + + import importlib.util + + spec = importlib.util.spec_from_file_location( + "check_uk_candidate_pair_pin", + Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + pair = make_pair(model="2.200.0") + resolved = cli.ResolvedCandidate( + pair, + cli.WheelFile("m.whl", "https://example.org/m.whl", pair.model.wheel_sha256), + cli.WheelFile("c.whl", "https://example.org/c.whl", pair.core.wheel_sha256), + "hf://model/example@pin/manifest.json", + ">=3.33.0", + ) + seen = {} + monkeypatch.setattr( + cli, + "resolve_candidate", + lambda **kwargs: seen.update(kwargs) or resolved, + ) + monkeypatch.setattr(cli, "git", lambda *args: "a" * 40) + outputs = [] + monkeypatch.setattr(cli, "set_output", lambda **kwargs: outputs.append(kwargs)) + out = tmp_path / "candidate.json" + + def args(pair_id): + return SimpleNamespace( + model_version="2.200.0", + data_version="1.58.0", + core_version="3.34.1", + pair_id=pair_id, + ledger=str(tmp_path), + force=False, + out=str(out), + ) + + assert cli.cmd_resolve(args("0" * 64)) == 1 + assert seen["core_version"] == "3.34.1" + assert outputs[-1]["action"] == "mismatch" + assert not out.exists() + assert "not the requested" in capsys.readouterr().out + assert cli.cmd_resolve(args(pair.pair_id)) == 0 + assert json.loads(out.read_text())["pair_id"] == pair.pair_id + + def test_resolution_picks_newest_stable_and_keeps_an_accepted_core(): resolved, manifest_bytes = _resolver() pair = resolved.pair @@ -201,6 +271,223 @@ def test_actions(tmp_path): ) +def test_a_promotable_pair_on_a_superseded_baseline_requalifies_once_per_commit( + tmp_path, +): + """Automatic promotion needs a qualification against main's current + certificate, so a promotable pair that advances main but compared + against an earlier one is requalified without force, once per main + commit. Nothing else is.""" + + pair = _promotable_pair_and_branch() + current = previous_certificate(BUNDLE) + earlier = current.model_copy(update={"bundle_version": "earlier"}) + pair_dir = tmp_path / "uk" / pair.pair_id + + def publish(receipt, certificate=earlier): + receipt = receipt.model_copy(update={"previous_certificate": certificate}) + write_json(pair_dir / "qualification_receipt.json", receipt.canonical_payload()) + + def action(base): + return candidate_action(pair, BUNDLE, tmp_path, base_commit=base * 40)[0] + + qualified = make_receipt(pair=pair) + publish(qualified) + verdict, reason = candidate_action(pair, BUNDLE, tmp_path, base_commit="b" * 40) + assert verdict == "qualify" and "superseded baseline" in reason + (pair_dir / "attempts" / ("b" * 12)).mkdir(parents=True) + assert action("b") == "duplicate" + assert action("c") == "qualify" + needs_review = make_receipt( + pair=pair, gate_results=gates(metrics_review="needs_review") + ).model_copy(update={"previous_certificate": earlier}) + accepted = accept_review( + needs_review, + reviewer="reviewer", + accepted_receipt_sha256=needs_review.unreviewed_sha256, + explanation="expected", + accepted_at="2026-10-16T09:00:00Z", + ) + assert accepted.promotable + publish(accepted) + assert action("c") == "qualify" + # Against main's own certificate, on main's own commit, or not + # promotable: nothing to do. + write_json( + pair_dir / "latest_attempt.json", {"attempt_key": "c" * 12, "sequence": 3} + ) + publish(qualified, current) + assert action("c") == "duplicate" + for results in (gates(metrics_review="needs_review"), gates(wheel_hashes="fail")): + publish(make_receipt(pair=pair, gate_results=results)) + assert action("c") == "duplicate" + # A pair that does not advance main in every component is never promoted. + sideways = make_pair( + model="2.200.0", + data="1.50.0", + core=BUNDLE["packages"]["policyengine-core"]["version"], + ) + write_json( + tmp_path / "uk" / sideways.pair_id / "qualification_receipt.json", + make_receipt(pair=sideways) + .model_copy(update={"previous_certificate": earlier}) + .canonical_payload(), + ) + assert ( + candidate_action(sideways, BUNDLE, tmp_path, base_commit="c" * 40)[0] + == "duplicate" + ) + + +def test_a_target_built_on_an_earlier_main_requalifies_one_attempt_at_a_time( + tmp_path, +): + """Promotion fast-forwards main to the qualified commit, so a promotable + pair built on an earlier main commit is requalified on the current one: + once per main commit, and never while another requalification runs.""" + + from policyengine.provenance.qualification_ledger import attempt_directory + from policyengine.provenance.uk_qualification import ( + requalification_target, + stage_attempt, + ) + + pair = _promotable_pair_and_branch() + pair_dir = tmp_path / "uk" / pair.pair_id + qualified = _promotion_receipt(pair=pair) + _publish_as_attempt(tmp_path, pair, qualified, "a" * 12) + write_json( + pair_dir / "latest_attempt.json", {"attempt_key": "a" * 12, "sequence": 0} + ) + now = "2026-10-16T10:00:00Z" + + def action(base): + return candidate_action(pair, BUNDLE, tmp_path, base_commit=base * 40, now=now) + + assert action("a")[0] == "duplicate" + verdict, reason = action("b") + assert verdict == "qualify" and "qualified on main aaaaaaaaaaaa" in reason + assert requalification_target([qualified], BUNDLE, tmp_path, "b" * 40) == pair + assert requalification_target([qualified], BUNDLE, tmp_path, "a" * 40) is None + # A requalification from b runs; c arrives meanwhile and waits for it. + stage_attempt( + attempt_directory(tmp_path, "uk", pair.pair_id, "b" * 12), + {"pair": pair.canonical_payload(), "pair_id": pair.pair_id}, + HARNESS, + owner_run_id="7", + staged_at="2026-10-16T09:30:00Z", + ) + assert action("b")[0] == "duplicate" + verdict, reason = action("c") + assert verdict == "duplicate" and "in flight" in reason + # Once it has finished (its receipt published), c requalifies. + requalified = qualified.model_copy(update={"completed_at": "2026-10-16T11:00:00Z"}) + _publish_as_attempt(tmp_path, pair, requalified, "b" * 12) + write_json( + pair_dir / "latest_attempt.json", {"attempt_key": "b" * 12, "sequence": 1} + ) + assert action("b")[0] == "duplicate" + assert action("c")[0] == "qualify" + # A repeat key on the same base names the same base. + write_json( + pair_dir / "latest_attempt.json", + {"attempt_key": "c" * 12 + "-2", "sequence": 2}, + ) + assert requalification_target([requalified], BUNDLE, tmp_path, "c" * 40) is None + + +def test_a_pair_without_an_attempt_pointer_takes_its_base_from_its_attempt(tmp_path): + """Round 2 of #635: a pair published before latest_attempt.json existed + must not be stranded. Its base is the attempt whose own receipt is the + published one; with no such attempt the base is unknown and stale.""" + + from policyengine.provenance.uk_qualification import ( + qualified_base, + requalification_target, + stale_qualification, + ) + + pair = _promotable_pair_and_branch() + pair_dir = tmp_path / "uk" / pair.pair_id + qualified = _promotion_receipt(pair=pair) + write_json(pair_dir / "qualification_receipt.json", qualified.canonical_payload()) + assert qualified_base(pair_dir) is None + assert "unrecorded" in stale_qualification(qualified, BUNDLE, pair_dir, "a" * 40) + assert requalification_target([qualified], BUNDLE, tmp_path, "a" * 40) == pair + _publish_as_attempt(tmp_path, pair, qualified, "a" * 12 + "-2") + assert qualified_base(pair_dir) == "a" * 12 + assert requalification_target([qualified], BUNDLE, tmp_path, "a" * 40) is None + assert requalification_target([qualified], BUNDLE, tmp_path, "b" * 40) == pair + + +def test_runs_resolved_on_different_main_commits_never_both_stage(tmp_path): + """Round 2 of #635: one attempt per pair at a time, whatever its base.""" + + from policyengine.provenance.qualification_ledger import attempt_directory + from policyengine.provenance.uk_qualification import stage_attempt + + pair = make_pair(model="2.200.0") + now = "2026-10-16T10:00:00Z" + assert ( + candidate_action(pair, BUNDLE, tmp_path, base_commit="a" * 40, now=now)[0] + == "qualify" + ) + stage_attempt( + attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12), + {"pair": pair.canonical_payload(), "pair_id": pair.pair_id}, + HARNESS, + owner_run_id="7", + staged_at="2026-10-16T09:30:00Z", + ) + verdict, reason = candidate_action( + pair, BUNDLE, tmp_path, base_commit="b" * 40, now=now + ) + assert verdict == "duplicate" and "in flight" in reason + # Abandoned (past its expiry), it no longer holds the pair. + later = "2026-10-16T14:00:00Z" + assert ( + candidate_action(pair, BUNDLE, tmp_path, base_commit="b" * 40, now=later)[0] + == "qualify" + ) + + +def test_requalify_names_the_target_only_while_its_baseline_is_stale( + monkeypatch, tmp_path, capsys +): + import argparse + import importlib.util + + spec = importlib.util.spec_from_file_location( + "check_uk_candidate_requalify", + Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + current = _promotion_receipt(pair=_promotable_pair_and_branch()) + stale = current.model_copy( + update={ + "previous_certificate": current.previous_certificate.model_copy( + update={"bundle_version": "earlier"} + ) + } + ) + monkeypatch.setattr(cli, "git", lambda *args: "a" * 40) + write_json( + tmp_path / "uk" / current.pair_id / "latest_attempt.json", + {"attempt_key": "a" * 12, "sequence": 1}, + ) + for receipts, expected in (([stale], [stale.pair]), ([current], []), ([], [])): + monkeypatch.setattr(cli, "latest_receipts", lambda root, r=receipts, **_: r) + assert cli.cmd_requalify(argparse.Namespace(ledger=str(tmp_path))) == 0 + printed = [ + json.loads(line) for line in capsys.readouterr().out.splitlines() if line + ] + assert printed == [ + {"pair": pair.canonical_payload(), "pair_id": pair.pair_id} + for pair in expected + ] + + def test_incomplete_pairs_requalify_on_a_new_base_only(tmp_path): pair = make_pair(model="2.200.0") incomplete = make_receipt( @@ -640,12 +927,382 @@ def test_a_branch_that_moved_off_the_qualified_commit_is_not_promoted(): assert plan.ready_branch == branch -def test_promote_readies_and_merges_only_the_qualified_commit(monkeypatch, tmp_path): +CODE = ".github/workflows/pr_code_changes.yaml" + + +def _check(name, status="completed", conclusion="success", id=1): + return {"id": id, "name": name, "status": status, "conclusion": conclusion} + + +def _run(event, status="completed", conclusion="success", id=1, path=CODE): + return { + "id": id, + "path": path, + "event": event, + "status": status, + "conclusion": conclusion, + } + + +CHECKS_PASSED = [ + _check("Lint", id=1), + _check("Test (3.13)", id=2), + _check("Docs", conclusion="skipped", id=3), +] +RUNS_PASSED = [_run("pull_request", id=10), _run("workflow_dispatch", id=11)] + + +def _gh_api_output(argv, checks, runs, commit, branch): + """What the fake `gh api` prints: the call's own jq applied by the real + jq to an API-shaped page. Each run is the candidate's own (its commit, + branch and this repository) unless it says otherwise, with its repository + as the API's object; each check's suite id sits under `check_suite`, as + in the API (rounds 5 and 6 of #635: fakes that returned every field, or + picked fields by name, let a dropped field or a wrong source path pass). + None for an unreadable list.""" + + if "check-runs" in argv[2]: + if checks is None: + return None + page = { + "check_runs": [ + { + **{k: v for k, v in check.items() if k != "check_suite_id"}, + "check_suite": {"id": check.get("check_suite_id")}, + } + for check in checks + ] + } + else: + if runs is None: + return None + page = { + "workflow_runs": [ + { + "head_sha": commit, + "head_branch": branch, + **run, + "head_repository": { + "full_name": run.get( + "head_repository", "PolicyEngine/policyengine.py" + ) + }, + } + for run in runs + ] + } + return jq(argv[argv.index("--jq") + 1], page) + + +def _gh_pr_list_output(argv, listing): + """What the fake `gh pr list` prints, filtering as gh does: by state + (open unless the call says), base, head branch, a `head:` search prefix, + then the limit (30 unless the call names one); each pull request cut to + the `--json` fields, and the call's `--jq` applied by the real jq. A + listing item is open and to main unless it says otherwise.""" + + def option(name, default=None): + return argv[argv.index(name) + 1] if name in argv else default + + state = option("--state", "open") + base = option("--base") + head = option("--head") + search = option("--search", "") + prefix = search.removeprefix("head:") if search.startswith("head:") else None + fields = set(option("--json").split(",")) + matched = [ + pr + for pr in ({"state": "OPEN", "baseRefName": "main", **pr} for pr in listing) + if (state == "all" or pr["state"] == state.upper()) + and (base is None or pr["baseRefName"] == base) + and (head is None or pr["headRefName"] == head) + and (prefix is None or pr["headRefName"].startswith(prefix)) + ][: int(option("--limit", "30"))] + printed = [{k: v for k, v in pr.items() if k in fields} for pr in matched] + expression = option("--jq") + return jq(expression, printed) if expression else json.dumps(printed) + + +# Each: checks, runs, branch at the commit, main ("parent" or "moved"), the +# push's exit code, automatic; then whether it is promoted, and the readiness +# calls made ("ready" or "undo") in order. +PROMOTION_CASES = { + "everything passed": ( + CHECKS_PASSED, + RUNS_PASSED, + True, + "parent", + 0, + True, + True, + ["ready"], + ), + "a test still running": ( + [*CHECKS_PASSED[:1], _check("Test (3.13)", "in_progress", None, id=4)], + RUNS_PASSED, + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "a test failed": ( + [*CHECKS_PASSED[:1], _check("Test (3.13)", conclusion="failure", id=4)], + RUNS_PASSED, + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "an older cancelled run a newer success replaced": ( + [*CHECKS_PASSED, _check("Test (3.13)", conclusion="cancelled", id=0)], + # The API lists newest first. + [*RUNS_PASSED, _run("pull_request", conclusion="cancelled", id=9)], + True, + "parent", + 0, + True, + True, + ["ready"], + ), + "a newer failure after an older success": ( + [*CHECKS_PASSED, _check("Test (3.13)", conclusion="failure", id=9)], + RUNS_PASSED, + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "no run on the candidate's own tree": ( + CHECKS_PASSED, + RUNS_PASSED[:1], + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "the own-tree run still registering its jobs": ( + CHECKS_PASSED[:1], + [RUNS_PASSED[0], _run("workflow_dispatch", "in_progress", None, id=11)], + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "no checks yet": ([], RUNS_PASSED, True, "parent", 0, True, False, ["undo"]), + # Round 3 of #635: a dispatch cancelled while queued has no jobs and so + # no checks; the run itself must still block. + "the own-tree run cancelled with no jobs": ( + CHECKS_PASSED, + [RUNS_PASSED[0], _run("workflow_dispatch", conclusion="cancelled", id=11)], + True, + "parent", + 0, + True, + False, + ["undo"], + ), + # Round 3 of #635: a replaced run re-run later gets newer check ids; its + # checks must not count, whatever they conclude. + "a replaced run's re-run checks": ( + [ + {**_check("Lint", id=1), "check_suite_id": 100}, + {**_check("Test (3.13)", id=2), "check_suite_id": 100}, + {**_check("Own tree", id=3), "check_suite_id": 110}, + { + **_check("Test (3.13)", conclusion="failure", id=50), + "check_suite_id": 90, + }, + ], + [ + {**_run("pull_request", id=10), "check_suite_id": 100}, + {**_run("workflow_dispatch", id=11), "check_suite_id": 110}, + {**_run("pull_request", conclusion="failure", id=9), "check_suite_id": 90}, + ], + True, + "parent", + 0, + True, + True, + ["ready"], + ), + # An app's check outside any workflow run still counts. + "an app's failing check": ( + [ + {**_check("Lint", id=1), "check_suite_id": 100}, + {**_check("External", conclusion="failure", id=5), "check_suite_id": 500}, + ], + [ + {**_run("pull_request", id=10), "check_suite_id": 100}, + {**_run("workflow_dispatch", id=11), "check_suite_id": 110}, + ], + True, + "parent", + 0, + True, + False, + ["undo"], + ), + # Round 5 of #635: the runs listed at a commit include any other pull + # request's at it, a fork's included. A newer failing run of a fork, or of + # another branch here, and its checks neither block the candidate ... + "a fork's newer failing run at the commit": ( + [ + {**_check("Lint", id=1), "check_suite_id": 100}, + {**_check("Own tree", id=3), "check_suite_id": 110}, + {**_check("Lint", conclusion="failure", id=60), "check_suite_id": 200}, + ], + [ + {**_run("pull_request", id=10), "check_suite_id": 100}, + {**_run("workflow_dispatch", id=11), "check_suite_id": 110}, + { + **_run("pull_request", conclusion="failure", id=20), + "check_suite_id": 200, + "head_repository": "someone/policyengine.py", + }, + ], + True, + "parent", + 0, + True, + True, + ["ready"], + ), + "another branch's newer failing run at the commit": ( + [ + {**_check("Lint", id=1), "check_suite_id": 100}, + {**_check("Own tree", id=3), "check_suite_id": 110}, + {**_check("Lint", conclusion="failure", id=60), "check_suite_id": 200}, + ], + [ + {**_run("pull_request", id=10), "check_suite_id": 100}, + {**_run("workflow_dispatch", id=11), "check_suite_id": 110}, + { + **_run("pull_request", conclusion="failure", id=20), + "check_suite_id": 200, + "head_branch": "candidate/uk/other", + }, + ], + True, + "parent", + 0, + True, + True, + ["ready"], + ), + # ... nor stand in for a run of its own. + "only a fork's run on the candidate's tree": ( + [ + {**_check("Lint", id=1), "check_suite_id": 100}, + {**_check("Own tree", id=3), "check_suite_id": 200}, + ], + [ + {**_run("pull_request", id=10), "check_suite_id": 100}, + { + **_run("workflow_dispatch", id=20), + "check_suite_id": 200, + "head_repository": "someone/policyengine.py", + }, + ], + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "the checks could not be read": ( + None, + RUNS_PASSED, + True, + "parent", + 0, + True, + False, + ["undo"], + ), + "the branch moved": ( + CHECKS_PASSED, + RUNS_PASSED, + False, + "parent", + 0, + True, + False, + ["undo"], + ), + "main moved since the base": ( + CHECKS_PASSED, + RUNS_PASSED, + True, + "moved", + 0, + True, + False, + ["undo"], + ), + "main moved at the push": ( + CHECKS_PASSED, + RUNS_PASSED, + True, + "parent", + 1, + True, + False, + ["ready", "undo"], + ), + "manual promotion, on the base": ( + None, + None, + True, + "parent", + 0, + False, + False, + ["ready"], + ), + "manual promotion, main moved": ( + None, + None, + True, + "moved", + 0, + False, + False, + ["undo"], + ), +} + + +@pytest.mark.parametrize("case", sorted(PROMOTION_CASES)) +def test_promotion_makes_main_exactly_the_qualified_commit( + monkeypatch, tmp_path, capsys, case +): + """Round 2 of #635. A candidate is readied only while main is still the + commit it was built on, so no merge can combine it with other code. + Automatic promotion waits for the commit's own tree to pass its checks + (the newest run of each workflow, the newest check of each name), then + moves main to it with a push leased on that parent.""" + import argparse import importlib.util + checks, runs, at_commit, main, push_code, automatic, promotes, readiness = ( + PROMOTION_CASES[case] + ) qualified = _promotion_receipt(pair=_promotable_pair_and_branch()) branch = candidate_branch(qualified.pair) + commit = qualified.harness.git_commit + parent = "p" * 40 spec = importlib.util.spec_from_file_location( "check_uk_candidate_for_test", Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", @@ -653,22 +1310,703 @@ def test_promote_readies_and_merges_only_the_qualified_commit(monkeypatch, tmp_p cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) calls = [] + + target = { + "number": 1, + "headRefName": branch, + "headRefOid": commit, + "isDraft": True, + "isCrossRepository": False, + } + + def run(argv, **kwargs): + calls.append(argv) + out = "" + code = 0 + if argv[:3] == ["gh", "api", "graphql"]: + out = graphql_base_changes_output(argv, set()) + elif argv[:2] == ["gh", "api"] and argv[4:5] == [f"{_REPO}/pulls"]: + out = rest_pulls_output(argv, [target]) + elif argv[:2] == ["gh", "api"]: + printed = _gh_api_output(argv, checks, runs, commit, branch) + if printed is None: + code = 1 + else: + out = printed + elif argv[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + out = commit if at_commit else "f" * 40 + elif argv[:3] == ["git", "rev-parse", f"{commit}^"]: + out = parent + elif argv[:2] == ["git", "ls-remote"]: + out = f"{parent if main == 'parent' else 'm' * 40}\trefs/heads/main\n" + elif argv[:2] == ["git", "push"]: + code = push_code + return SimpleNamespace(returncode=code, stdout=out, stderr="") + monkeypatch.setattr(cli, "latest_receipts", lambda root, **_: [qualified]) - monkeypatch.setattr( - cli, "subprocess", SimpleNamespace(run=lambda argv, **k: calls.append(argv)) + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=run)) + monkeypatch.setattr(cli, "config", lambda: {"promotion": {"automatic": automatic}}) + monkeypatch.setattr(cli, "_candidate_pull_requests", lambda: [target]) + assert cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) == 0 + assert [ + "undo" if "--undo" in argv else "ready" + for argv in calls + if argv[:3] == ["gh", "pr", "ready"] + ] == readiness + pushes = [argv for argv in calls if argv[:2] == ["git", "push"]] + reaches_push = "ready" in readiness and automatic + assert pushes == ( + [ + [ + "git", + "push", + f"--force-with-lease=refs/heads/main:{parent}", + "origin", + f"{commit}:refs/heads/main", + ] + ] + if reaches_push + else [] ) + assert not [argv for argv in calls if argv[:3] == ["gh", "pr", "merge"]] + if not automatic: + assert not [argv for argv in calls if argv[:2] == ["gh", "api"]] + assert (f"promoted {branch}" in capsys.readouterr().out) is promotes + + +@pytest.mark.parametrize("undo_fails", [False, True]) +def test_promotion_returns_every_other_ready_candidate_to_draft( + monkeypatch, tmp_path, undo_fails +): + """Round 3 of #635: only the plan's target may be ready, so a candidate + left ready by a run whose undo failed, or by older code, is returned to + draft; a failed return to draft fails the run so it is seen.""" + + import argparse + + cli, calls, branch = _sweep_cli( + monkeypatch, path="main moved", failing={"2", "1"} if undo_fails else set() + ) + status = cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) + undone = [argv[3] for argv in calls if argv[:3] == ["gh", "pr", "ready"]] + # The stale one (the draft and the fork's are left alone), then the target + # by _promote (main moved), each by number. + assert undone == ["2", "1"] + assert status == (1 if undo_fails else 0) + + +SUPERSEDED_PAIR = make_pair(model="2.0.0", data="1.0.0", core="3.0.0") +_REPO = "repos/PolicyEngine/policyengine.py" + + +def _sweep_cli( + monkeypatch, + *, + path, + failing, + checks=None, + runs=None, + push_code=1, + extra=(), + superseded=False, + shared_fails=None, + base_changed=(), +): + """The promote CLI over a real `gh pr list` payload: the target (#1, + ready), a stale same-repository candidate (#2, ready), a draft (#3), a + fork's pull request with the stale branch's name (#4, ready), a fork's + with the target's branch name at another commit (#5, ready) and a ready + one whose listing lacks the fork field (#6), then ``extra`` (each may + use "{branch}" and "{commit}" for the target's). ``superseded`` adds a + receipt for an older pair with an open pull request (#9), which the plan + closes. ``failing`` names the pull request numbers whose undo fails; + ``path`` is "main moved", "lease rejected" or "main at the parent"; + ``base_changed`` names the pull requests whose base was ever changed; + ``shared_fails`` fails the REST listing of a branch's pull requests + ("listing") or the base-change query ("history").""" + + import importlib.util + + qualified = _promotion_receipt(pair=_promotable_pair_and_branch()) + branch = candidate_branch(qualified.pair) + commit = qualified.harness.git_commit + receipts = [qualified] + if superseded: + receipts.append(_promotion_receipt(pair=SUPERSEDED_PAIR)) + stale = "candidate/uk/stale-pair" + spec = importlib.util.spec_from_file_location( + "check_uk_candidate_for_test", + Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + calls = [] + listing = [ + { + "number": 1, + "headRefName": branch, + "headRefOid": commit, + "isDraft": False, + "isCrossRepository": False, + }, + { + "number": 2, + "headRefName": stale, + "headRefOid": "s" * 40, + "isDraft": False, + "isCrossRepository": False, + }, + { + "number": 3, + "headRefName": "candidate/uk/draft", + "headRefOid": "d" * 40, + "isDraft": True, + "isCrossRepository": False, + }, + { + "number": 4, + "headRefName": stale, + "headRefOid": "f" * 40, + "isDraft": False, + "isCrossRepository": True, + }, + { + "number": 5, + "headRefName": branch, + "headRefOid": "c" * 40, + "isDraft": False, + "isCrossRepository": True, + }, + { + "number": 6, + "headRefName": "candidate/uk/unknown", + "headRefOid": "u" * 40, + "isDraft": False, + }, + *( + { + key: value.format(branch=branch, commit=commit) + if isinstance(value, str) + else value + for key, value in pr.items() + } + for pr in extra + ), + ] + if superseded: + listing.append( + { + "number": 9, + "headRefName": candidate_branch(SUPERSEDED_PAIR), + "headRefOid": "o" * 40, + "isDraft": True, + "isCrossRepository": False, + } + ) + + def run(argv, **kwargs): + calls.append(argv) + out, code = "", 0 + if argv[:3] == ["gh", "pr", "list"]: + out = _gh_pr_list_output(argv, listing) + elif argv[:3] == ["gh", "api", "graphql"]: + if shared_fails == "history": + code = 1 + else: + out = graphql_base_changes_output(argv, base_changed) + elif argv[:2] == ["gh", "api"] and argv[4:5] == [f"{_REPO}/pulls"]: + if shared_fails == "listing": + code = 1 + else: + out = rest_pulls_output(argv, listing) + elif argv[:2] == ["gh", "api"]: + out = _gh_api_output( + argv, checks or CHECKS_PASSED, runs or RUNS_PASSED, commit, branch + ) + elif argv[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + out = commit + elif argv[:3] == ["git", "rev-parse", f"{commit}^"]: + out = "p" * 40 + elif argv[:2] == ["git", "ls-remote"]: + main = "m" * 40 if path == "main moved" else "p" * 40 + out = main + "\trefs/heads/main\n" + elif argv[:2] == ["git", "push"]: + code = push_code # 1: the lease is rejected (main moved at the push) + elif argv[:3] == ["gh", "pr", "ready"] and "--undo" in argv: + code = 1 if argv[3] in failing else 0 + return SimpleNamespace(returncode=code, stdout=out, stderr="") + + monkeypatch.setattr(cli, "latest_receipts", lambda root, **_: receipts) + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=run)) monkeypatch.setattr(cli, "config", lambda: {"promotion": {"automatic": True}}) - monkeypatch.setattr(cli, "_open_candidate_branches", lambda: {branch: "f" * 40}) + return cli, calls, branch + + +@pytest.mark.parametrize("path", ["main moved", "lease rejected"]) +@pytest.mark.parametrize("failing", [set(), {"2"}, {"1"}]) +def test_each_failed_return_to_draft_fails_the_run( + monkeypatch, tmp_path, path, failing +): + """Round 4 of #635: the sweep's failed undo and the target's failed undo + (whether main moved or the lease was rejected) each fail the run alone. + Round 5: every readiness change goes by number, on same-repository pull + requests only; a fork's with the target's branch name neither changes the + target's commit nor is touched, and one whose listing lacks the fork + field counts as a fork's.""" + + import argparse + + cli, calls, branch = _sweep_cli(monkeypatch, path=path, failing=failing) + candidates = cli._candidate_pull_requests() + assert [pr["number"] for pr in candidates] == [1, 2, 3] + assert {pr["headRefName"]: pr["headRefOid"] for pr in candidates}[branch] != ( + "c" * 40 + ) + status = cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) + readiness = [ + ("undo" if "--undo" in argv else "ready", argv[3]) + for argv in calls + if argv[:3] == ["gh", "pr", "ready"] + ] + target = ( + [("undo", "1")] if path == "main moved" else [("ready", "1"), ("undo", "1")] + ) + assert readiness == [("undo", "2"), *target] + assert status == (1 if failing else 0) + listing = next(argv for argv in calls if argv[:3] == ["gh", "pr", "list"]) + assert listing[listing.index("--limit") + 1] == str(cli.PULL_REQUEST_LIMIT) + + +@pytest.mark.parametrize(("listed", "refused"), [(999, False), (1000, True)]) +def test_a_candidate_listing_at_its_limit_is_refused(monkeypatch, listed, refused): + """Round 5 of #635: gh stops a pull request search at its limit, so a + listing that reaches it may have missed the target; promotion refuses to + go on rather than act on part of the list.""" + + import importlib.util + + spec = importlib.util.spec_from_file_location( + "check_uk_candidate_for_test", + Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + listing = [ + { + "number": number, + "headRefName": f"candidate/uk/pair-{number}", + "headRefOid": "a" * 40, + "isDraft": True, + "isCrossRepository": False, + } + for number in range(1, 1201) + ] + + def run(argv, **kwargs): + return SimpleNamespace( + returncode=0, stdout=_gh_pr_list_output(argv, listing[:listed]), stderr="" + ) + + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=run)) + assert cli.PULL_REQUEST_LIMIT == 1000 + if refused: + with pytest.raises(SystemExit, match="::error::1000 open pull requests"): + cli._candidate_pull_requests() + else: + assert len(cli._candidate_pull_requests()) == listed + + +OTHER_BASE = { + "number": 7, + "headRefName": "{branch}", + "headRefOid": "{commit}", + "baseRefName": "release", + "isDraft": False, + "isCrossRepository": False, +} + + +def _other(**fields): + return {**OTHER_BASE, **fields} + + +FORKS = [ + _other(number=100 + n, baseRefName="main", isCrossRepository=True) + for n in range(40) +] + +# Each: other pull requests from the target's branch, the numbers whose base +# was ever changed, whether promotion refuses the target, and the pull +# request the refusal names. +SHARED_BRANCH_CASES = { + "open, to another base": ([_other()], (), True, 7), + "open, to another base, its head not yet updated": ( + [_other(headRefOid="a" * 40)], + (), + True, + 7, + ), + "closed at the commit, to another base": ([_other(state="CLOSED")], (), True, 7), + "merged at the commit, to another base": ([_other(state="MERGED")], (), True, 7), + "closed at another commit, to another base": ( + [_other(state="CLOSED", headRefOid="a" * 40)], + (), + False, + None, + ), + "a fork's, open, to another base": ( + [_other(isCrossRepository=True)], + (), + False, + None, + ), + # Round 9 of #635: a base branch renamed into main records no event, so + # a pull request now to main proves nothing about what its runs at the + # commit tested; any other pull request from the branch at the commit + # counts, whatever its base. (Prepare's own pull request to main is + # always at the commit too, so a renamed one sits beside it.) + "closed at the commit, to main (a rename leaves no event)": ( + [_other(state="CLOSED", baseRefName="main")], + (), + True, + 7, + ), + "merged at the commit, to main": ( + [_other(state="MERGED", baseRefName="main")], + (), + True, + 7, + ), + "closed at another commit, to main": ( + [_other(state="CLOSED", baseRefName="main", headRefOid="a" * 40)], + (), + False, + None, + ), + # Round 7 of #635: a base change does not change what a pull request's + # earlier runs tested, so one now to main that ever had another base + # counts, the candidate's own included. + "closed at the commit, to main after a base change": ( + [_other(state="CLOSED", baseRefName="main")], + (7,), + True, + 7, + ), + "closed at another commit, to main after a base change": ( + [_other(state="CLOSED", baseRefName="main", headRefOid="a" * 40)], + (7,), + False, + None, + ), + "the candidate's own, to main after a base change": ([], (1,), True, 1), + # Round 8: GitHub retargets a pull request itself when its base branch is + # merged and deleted, and records that as a separate event type. + "the candidate's own, to main after an automatic base change": ( + [], + {1: "automatic"}, + True, + 1, + ), + "closed at the commit, to main after an automatic base change": ( + [_other(state="CLOSED", baseRefName="main")], + {7: "automatic"}, + True, + 7, + ), + # Round 7: forks' pull requests with the branch's name, however many, + # neither crowd out nor stand in for this repository's. + "forty forks', to main, with its branch name": (FORKS, (), False, None), + "forty forks' and one to another base": (FORKS + [_other()], (), True, 7), +} + + +def _run_reads(calls): + """The calls that read a commit's runs or checks.""" + + return [ + argv + for argv in calls + if argv[:2] == ["gh", "api"] + and ("actions/runs" in argv[2] or "check-runs" in argv[2]) + ] + + +@pytest.mark.parametrize("case", sorted(SHARED_BRANCH_CASES)) +def test_a_candidate_whose_runs_may_have_tested_another_base_is_refused( + monkeypatch, tmp_path, capsys, case +): + """Round 6 of #635: a pull request from the candidate branch to another + base runs the same workflows with the same commit, branch and repository, + and GitHub's runs do not say which pull request they are for, so its + failure could strand the candidate and its success hide the candidate's + failure. While one is open, or after one closed at this commit, the + candidate is kept draft and the run fails with an error naming it; the + other pull request is never readied, undone or counted as a candidate. + Round 7: the same holds for a pull request now to main whose base was + ever changed, and forks' pull requests cannot crowd out the listing of + this repository's.""" + + import argparse + + extra, base_changed, refused, named = SHARED_BRANCH_CASES[case] + cli, calls, branch = _sweep_cli( + monkeypatch, + path="main at the parent", + failing=set(), + push_code=0, + extra=extra, + base_changed=base_changed, + ) + assert 7 not in [pr["number"] for pr in cli._candidate_pull_requests()] + status = cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) + output = capsys.readouterr().out + pushes = [argv for argv in calls if argv[:2] == ["git", "push"]] + readiness = [argv[3] for argv in calls if argv[:3] == ["gh", "pr", "ready"]] + assert "7" not in readiness + if refused: + assert status == 1 and not pushes + assert f"::error::{branch} is not promoted" in output + assert f"#{named} (" in output + assert readiness == ["2", "1"] + assert not _run_reads(calls) + else: + assert status == 0 and len(pushes) == 1 + assert f"promoted {branch}" in output + + +@pytest.mark.parametrize("how", ["listing", "history"]) +def test_unreadable_pull_requests_from_the_candidate_branch_wait( + monkeypatch, tmp_path, capsys, how +): + """Round 6 of #635: when the candidate branch's pull requests (round 7: + or a base-change history) cannot be read, the candidate waits as a draft + without reading its runs.""" + + import argparse + + cli, calls, branch = _sweep_cli( + monkeypatch, + path="main at the parent", + failing=set(), + push_code=0, + shared_fails=how, + ) assert cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) == 0 - assert calls == [] + output = capsys.readouterr().out + assert "could not be read" in output and "::error::" not in output + assert not [argv for argv in calls if argv[:2] == ["git", "push"]] + assert not _run_reads(calls) + assert [argv[3] for argv in calls if argv[:3] == ["gh", "pr", "ready"]] == [ + "2", + "1", + ] + + +def test_only_pull_requests_to_main_are_candidates(monkeypatch): + """Round 6 of #635: the listing asks gh for pull requests to main and + drops any other it returns, so two pull requests from one candidate + branch cannot collapse into one entry that sends readiness changes to + the wrong one.""" + + import importlib.util + + spec = importlib.util.spec_from_file_location( + "check_uk_candidate_for_test", + Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + calls = [] + listing = [ + { + "number": number, + "baseRefName": base, + "headRefName": "candidate/uk/2.200.0-1.58.9", + "headRefOid": "a" * 40, + "isDraft": False, + "isCrossRepository": False, + } + for number, base in ((1, "release"), (2, "main")) + ] + + def run(argv, **kwargs): + calls.append(argv) + # A gh that ignored --base: the script still keeps only main's. + return SimpleNamespace(returncode=0, stdout=json.dumps(listing), stderr="") + + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=run)) + assert [pr["number"] for pr in cli._candidate_pull_requests()] == [2] + assert calls[0][calls[0].index("--base") + 1] == "main" + + +def test_superseded_candidates_are_closed_by_number(monkeypatch, tmp_path): + """Round 6 of #635: closing a superseded candidate goes by its number, + like every other write to a pull request.""" + + import argparse + + cli, calls, branch = _sweep_cli( + monkeypatch, + path="main at the parent", + failing=set(), + push_code=0, + superseded=True, + ) + assert cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) == 0 + assert [argv[3] for argv in calls if argv[:3] == ["gh", "pr", "close"]] == ["9"] + + +NOW = "2026-10-16T12:00:00Z" +RECENT = "2026-10-16T06:00:00Z" +OLD = "2026-09-15T11:00:00Z" + +# Each: the own-tree run's status, conclusion, attempt and creation time; +# whether the candidate is stranded (waits on a run refresh will not re-run). +STRANDED_CASES = { + "failed, attempts left": ("completed", "failure", 3, RECENT, False), + "failed at the cap": ("completed", "failure", 4, RECENT, True), + "timed out, attempts left": ("completed", "timed_out", 1, RECENT, False), + "cancelled, attempts left": ("completed", "cancelled", 2, RECENT, False), + # Round 5 of #635: refresh re-runs only failed, timed-out and cancelled + # runs, and GitHub only within 30 days, so these wait on nothing. + "waiting on an approval": ("completed", "action_required", 1, RECENT, True), + "skipped": ("completed", "skipped", 1, RECENT, True), + "stale": ("completed", "stale", 1, RECENT, True), + "failed past GitHub's re-run window": ("completed", "failure", 1, OLD, True), + "failed just inside the window": ( + "completed", + "failure", + 1, + "2026-09-16T12:00:01Z", + False, + ), + # Round 6: GitHub re-runs a run exactly 30 days old, and not one a + # second older. + "failed exactly 30 days ago": ( + "completed", + "failure", + 1, + "2026-09-16T12:00:00Z", + False, + ), + "failed 30 days and a second ago": ( + "completed", + "failure", + 1, + "2026-09-16T11:59:59Z", + True, + ), + # Unfinished runs and successes are not stranded at any attempt. + "in progress at the cap": ("in_progress", None, 4, OLD, False), + "queued past the window": ("queued", None, 1, OLD, False), +} + + +@pytest.mark.parametrize("case", sorted(STRANDED_CASES)) +def test_a_candidate_waiting_on_runs_refresh_will_not_rerun_fails_the_run( + monkeypatch, tmp_path, capsys, case +): + """Round 4 of #635: refresh re-runs a run until MAX_RUN_ATTEMPTS, then + stops; a candidate still waiting on it would otherwise wait unseen, so + the promote run fails with an error naming it. Round 5: the same holds + for any finished, unsuccessful run refresh will not re-run (a result it + does not re-run, or one past GitHub's 30-day window). Otherwise it waits + quietly, kept draft.""" + + import argparse + import datetime + + status_, conclusion, attempt, created, stranded = STRANDED_CASES[case] + runs = [ + _run("pull_request", id=10), + { + **_run("workflow_dispatch", status_, conclusion, id=11), + "run_attempt": attempt, + "created_at": created, + }, + ] + cli, calls, branch = _sweep_cli( + monkeypatch, path="main at the parent", failing=set(), runs=runs + ) monkeypatch.setattr( cli, - "_open_candidate_branches", - lambda: {branch: qualified.harness.git_commit}, + "_now", + lambda: datetime.datetime.fromisoformat(NOW.replace("Z", "+00:00")), + ) + assert cli.MAX_RUN_ATTEMPTS == 4 + status = cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) + output = capsys.readouterr().out + assert status == (1 if stranded else 0) + assert ("::error::" in output) is stranded + if stranded: + assert branch in output and "will not re-run" in output + assert [argv for argv in calls if argv[:2] == ["git", "push"]] == [] + # The target waits as a draft (round 4: one a person left ready). + assert [argv[3] for argv in calls if argv[:3] == ["gh", "pr", "ready"]] == [ + "2", + "1", + ] + + +@pytest.mark.parametrize("other_waiting", [False, True]) +def test_a_success_on_the_last_attempt_is_not_stranded( + monkeypatch, tmp_path, capsys, other_waiting +): + """Round 5 of #635: a run that passes on its last allowed attempt is not + stranded, whether the candidate then promotes or still waits on another + run.""" + + import argparse + + runs = [ + _run("pull_request", "in_progress", None, id=10) + if other_waiting + else _run("pull_request", id=10), + {**_run("workflow_dispatch", id=11), "run_attempt": 4, "created_at": OLD}, + ] + cli, calls, branch = _sweep_cli( + monkeypatch, path="main at the parent", failing=set(), runs=runs, push_code=0 ) assert cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) == 0 - assert calls[0][:3] == ["gh", "pr", "ready"] and calls[0][3] == branch - assert calls[1][-2:] == ["--match-head-commit", qualified.harness.git_commit] + output = capsys.readouterr().out + assert "::error::" not in output + assert (f"promoted {branch}" in output) is not other_waiting + if other_waiting: + assert f"automatic promotion waits for {branch}" in output + + +def test_a_failed_return_to_draft_while_waiting_fails_the_run( + monkeypatch, tmp_path, capsys +): + """Round 5 of #635: on the waiting path the target goes back to draft; + if that fails, a ready pull request is left that a merge could combine + with other code, so the run fails (with nothing stranded).""" + + import argparse + + runs = [ + _run("pull_request", id=10), + _run("workflow_dispatch", "in_progress", None, id=11), + ] + cli, calls, branch = _sweep_cli( + monkeypatch, path="main at the parent", failing={"1"}, runs=runs + ) + assert cli.cmd_promote(argparse.Namespace(ledger=str(tmp_path), apply=True)) == 1 + output = capsys.readouterr().out + assert "::error::could not return 1 to draft" in output + assert "will not re-run" not in output + assert [argv for argv in calls if argv[:2] == ["git", "push"]] == [] + + +def test_automatic_promotion_is_on_with_its_basis_recorded(): + config = json.loads( + ( + Path(__file__).resolve().parents[1] + / "src/policyengine/data/qualification/uk.json" + ).read_text() + ) + assert config["promotion"]["automatic"] is True + assert "d1344" in config["promotion"]["decision"] def test_promotion_requires_the_current_supported_years_baseline(monkeypatch): @@ -756,7 +2094,7 @@ def gate(pair, years, previous_years): plan = plan_promotion([receipt_b, receipt_c], current_b, branches) assert plan.target == receipt_c assert plan.ready_branch is None and plan.ready_commit is None - assert "requalif" in plan.reason and "--force" in plan.reason + assert "requalif" in plan.reason and plan.requalify == receipt_c.pair @pytest.mark.parametrize( @@ -812,6 +2150,7 @@ def test_stale_baseline_does_not_close_or_fall_back_to_other_candidates(missing) assert plan.target == newest assert plan.ready_branch is None and plan.ready_commit is None assert plan.close_branches == [candidate_branch(superseded_by_main.pair)] + assert plan.requalify == newest.pair def test_promotion_on_an_unrelated_main_commit_keeps_the_same_baseline(): @@ -825,6 +2164,7 @@ def test_promotion_on_an_unrelated_main_commit_keeps_the_same_baseline(): ) assert plan.ready_branch == branch assert plan.ready_commit == qualified.harness.git_commit + assert plan.requalify is None def test_a_forced_rerun_stages_a_new_attempt(tmp_path): @@ -846,6 +2186,163 @@ def test_a_forced_rerun_stages_a_new_attempt(tmp_path): assert attempt_key(pair, tmp_path, base_commit=base, force=True) == f"{base[:12]}-3" +#: Source for the fake gh's REST pull request listing, embedded in the +#: workflow shell tests (they run gh in another process): `pulls(args, +#: listing)` answers `gh api -X GET repos//pulls -f head=: +#: -f base=... -f state=...` as GitHub does, keeping only pull requests from +#: that owner's branch (a fork's has another owner; an item's branch is the +#: BRANCH environment variable's unless it names another), and applies the +#: call's --jq to REST-shaped objects with the real jq. +FAKE_GH_PULLS = """ +import shutil +import subprocess + + +def pulls(args, listing): + fields = dict( + value.split("=", 1) for flag, value in zip(args, args[1:]) if flag == "-f" + ) + owner, _, branch = fields["head"].partition(":") + page = [] + for pr in listing: + pr = { + "headRefName": os.environ["BRANCH"], + "state": "OPEN", + "isDraft": True, + **pr, + } + pr_owner = "someone" if pr.get("isCrossRepository") else "PolicyEngine" + state = "open" if pr["state"] == "OPEN" else "closed" + if ( + pr_owner == owner + and pr["headRefName"] == branch + and pr["baseRefName"] == fields.get("base", pr["baseRefName"]) + and fields.get("state", "open") in ("all", state) + ): + page.append( + { + "number": pr["number"], + "state": state, + "draft": pr["isDraft"], + "base": {"ref": pr["baseRefName"]}, + "head": {"ref": pr["headRefName"]}, + } + ) + sys.stdout.write( + subprocess.run( + [shutil.which("jq"), "-r", args[args.index("--jq") + 1]], + input=json.dumps(page), + capture_output=True, + text=True, + check=True, + ).stdout + ) +""" + +#: Forks' pull requests with the candidate branch's name, newer than its own: +#: gh pr list's default listing of 30 would hold only these (round 7 of #635). +FORK_PULLS = [ + { + "number": 600 + n, + "baseRefName": "main", + "isCrossRepository": True, + "isDraft": False, + } + for n in range(40) +] + + +@pytest.mark.parametrize("step_name", ["open", "verdict"]) +@pytest.mark.parametrize("exists", [True, False]) +def test_the_workflow_writes_to_the_candidates_own_pull_request_by_number( + tmp_path, step_name, exists +): + """Round 6 of #635: the steps that open or update the candidate's pull + request find it among this repository's pull requests to main and write + by number; a fork's pull request with the branch's name, or this + repository's from the branch to another base, is never edited. With no + candidate pull request, the prepare step opens one to main and the + verdict step fails.""" + + import yaml + + assert shutil.which("jq"), "jq is required to run the workflow's jq" + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/certify-uk-release.yaml").read_text() + ) + job, name = { + "open": ("prepare", "Open or update the draft candidate PR"), + "verdict": ("replay", "Update the candidate PR with the verdict"), + }[step_name] + step = next(s for s in workflow["jobs"][job]["steps"] if s.get("name") == name) + log_path = tmp_path / "commands.jsonl" + listing = [ + *FORK_PULLS, + {"number": 576, "baseRefName": "release", "isCrossRepository": False}, + { + "number": 577, + "baseRefName": "main", + "isCrossRepository": False, + "state": "CLOSED", + }, + ] + if exists: + listing.append( + {"number": 574, "baseRefName": "main", "isCrossRepository": False} + ) + fake = f"""#!{sys.executable} +import json +import os +import sys +from pathlib import Path + +{FAKE_GH_PULLS} +args = sys.argv[1:] +with Path(os.environ["COMMAND_LOG"]).open("a") as log: + log.write(json.dumps(["gh", *args]) + "\\n") +if args[:1] == ["api"] and "repos/PolicyEngine/policyengine.py/pulls" in args: + pulls(args, {listing!r}) +""" + binary_dir = tmp_path / "bin" + binary_dir.mkdir() + (binary_dir / "gh").write_text(fake) + (binary_dir / "gh").chmod(0o755) + branch = "candidate/uk/2.126.8-1.58.0" + result = subprocess.run( + ["bash", "-e", "-o", "pipefail", "-c", step["run"]], + cwd=tmp_path, + env={ + **os.environ, + "PATH": f"{binary_dir}{os.pathsep}{os.environ['PATH']}", + "BRANCH": branch, + "PAIR_ID": "a" * 64, + "GITHUB_REPOSITORY": "PolicyEngine/policyengine.py", + "GITHUB_RUN_ID": "1", + "LEDGER_BRANCH": "uk-qualification-ledger", + "RUNNER_TEMP": str(tmp_path), + "COMMAND_LOG": str(log_path), + }, + capture_output=True, + text=True, + timeout=30, + ) + commands = [json.loads(line) for line in log_path.read_text().splitlines()] + writes = [c for c in commands if c[1:3] in (["pr", "edit"], ["pr", "create"])] + if exists: + assert result.returncode == 0, result.stdout + result.stderr + assert [c[:4] for c in writes] == [["gh", "pr", "edit", "574"]] + elif step_name == "open": + assert result.returncode == 0, result.stdout + result.stderr + assert len(writes) == 1 and writes[0][1:3] == ["pr", "create"] + create = writes[0] + assert create[create.index("--base") + 1] == "main" + assert create[create.index("--head") + 1] == branch + else: + assert result.returncode == 1 + assert f"no open pull request from {branch} to main" in result.stdout + assert writes == [] + + @pytest.mark.parametrize( ("pr_state", "reset_fails"), [("ready", False), ("draft", False), ("absent", False), ("ready", True)], @@ -874,15 +2371,36 @@ def test_the_workflow_drafts_existing_prs_before_replacing_their_head( import sys from pathlib import Path +{FAKE_GH_PULLS} command = Path(sys.argv[0]).name args = sys.argv[1:] state_path = Path(os.environ["PR_STATE"]) state = json.loads(state_path.read_text()) with Path(os.environ["COMMAND_LOG"]).open("a") as log: log.write(json.dumps([command, *args]) + "\\n") -if command == "gh" and args[:2] == ["pr", "list"]: - if state["status"] == "ready": - print("574") +if command == "gh" and args[:1] == ["api"]: + # Forty newer ready forks' pull requests with the branch's name, and one + # of this repository's from the branch to another base, are never undone + # and never hide the candidate's own (round 7 of #635). + listing = [ + *{FORK_PULLS!r}, + {{ + "number": 576, + "baseRefName": "release", + "isCrossRepository": False, + "isDraft": False, + }}, + ] + if state["status"] != "absent": + listing.append( + {{ + "number": 574, + "baseRefName": "main", + "isCrossRepository": False, + "isDraft": state["status"] != "ready", + }} + ) + pulls(args, listing) elif command == "gh" and args[:2] == ["pr", "ready"]: assert args[2:] == ["574", "--undo"] if os.environ["RESET_FAILS"] == "1": @@ -907,6 +2425,7 @@ def test_the_workflow_drafts_existing_prs_before_replacing_their_head( "PATH": f"{binary_dir}{os.pathsep}{os.environ['PATH']}", "BRANCH": "candidate/uk/2.126.8-1.58.0", "PAIR_ID": "a" * 64, + "GITHUB_REPOSITORY": "PolicyEngine/policyengine.py", "GH_TOKEN": "local-workflow-test", "PR_STATE": str(state_path), "COMMAND_LOG": str(log_path), @@ -1240,12 +2759,16 @@ def test_resolution_preserves_authorization_to_recover_forced_work( args = SimpleNamespace( model_version=None, data_version=None, + core_version=None, + pair_id=None, ledger=str(tmp_path), force=True, out=str(out), ) assert cli.cmd_resolve(args) == 0 payload = json.loads(out.read_text()) + # The recheck under the preparation lock reads it (round 3 of #635). + assert payload["force"] is True assert payload["supersedes_receipt_sha256"] == terminal.unreviewed_sha256 attempt = attempt_directory(tmp_path, "uk", pair.pair_id, payload["attempt_key"]) stage_attempt( @@ -1257,7 +2780,7 @@ def test_resolution_preserves_authorization_to_recover_forced_work( args.force = False assert cli.cmd_resolve(args) == 0 recovery = json.loads(out.read_text()) - assert recovery["action"] == "qualify" + assert recovery["action"] == "qualify" and recovery["force"] is False assert recovery["base_commit"] == base assert recovery["attempt_key"] == (base[:12] if main_advanced else base[:12] + "-2") assert recovery["supersedes_receipt_sha256"] == terminal.unreviewed_sha256 diff --git a/tests/test_uk_qualification_workflow.py b/tests/test_uk_qualification_workflow.py index 88b57b4c2..10f1c2a8b 100644 --- a/tests/test_uk_qualification_workflow.py +++ b/tests/test_uk_qualification_workflow.py @@ -19,7 +19,16 @@ candidate_branch, previous_certificate, ) -from tests.qualification_fixtures import gates, make_pair, make_receipt +from tests.qualification_fixtures import ( + HARNESS, + gates, + graphql_base_changes_output, + make_pair, + make_receipt, + rest_pulls_output, +) + +PULLS = ["repos/PolicyEngine/policyengine.py/pulls"] REPO_ROOT = Path(__file__).resolve().parents[1] WORKFLOW_PATH = REPO_ROOT / ".github/workflows/certify-uk-release.yaml" @@ -95,6 +104,127 @@ def test_every_uv_step_runs_the_environment_its_job_installed(): ] +def test_catch_up_requalifies_a_stale_target_after_finishing_receipts(): + """The requalification dispatch runs without force, after waiting + receipts are finished and published, so it reads the newest ledger.""" + + job = _workflow()["jobs"]["catch-up"] + runs = [step.get("run", "") for step in job["steps"]] + requalify = next( + index + for index, run in enumerate(runs) + if "check_uk_candidate.py requalify" in run + ) + published = next( + index + for index, run in enumerate(runs) + if "qualification-ledger.sh publish" in run + ) + assert published < requalify + assert '"-f", "force=false"' in runs[requalify] + assert "force=true" not in runs[requalify] + assert job["steps"][requalify]["env"]["GH_TOKEN"] == "${{ github.token }}" + assert job["permissions"]["actions"] == "write" + + +def test_every_dispatch_names_the_exact_pair(): + """Recovery and requalification dispatch one pair: its versions, its core + and its id, which resolve checks, so another pair is never qualified in + its place.""" + + workflow = _workflow() + inputs = workflow[True]["workflow_dispatch"]["inputs"] + assert {"core_version", "pair_id"} <= set(inputs) + resolve = next( + step + for step in workflow["jobs"]["resolve"]["steps"] + if step.get("name") == "Resolve and deduplicate the candidate" + ) + assert '--core-version "$INPUT_CORE"' in resolve["run"] + assert '--pair-id "$INPUT_PAIR_ID"' in resolve["run"] + dispatches = [ + step["run"] + for step in workflow["jobs"]["catch-up"]["steps"] + if 'gh", "workflow", "run' in step.get("run", "") + or '"gh", "workflow", "run"' in step.get("run", "") + ] + assert len(dispatches) == 2 + for run in dispatches: + assert "core_version={pair['core']['version']}" in run + assert "pair_id={pair_id}" in run + assert 'entry["pair_id"]' in run + + +def test_preparation_rechecks_eligibility_and_tests_the_candidates_own_tree(): + """Round 2 of #635: under the preparation lock, a run whose pair another + run is already qualifying (from any main commit) stops; and the + code-changes workflow runs on the candidate branch itself, the tree + promotion would put on main.""" + + job = _workflow()["jobs"]["prepare"] + steps = job["steps"] + names = [step.get("name") for step in steps] + duplicate = steps[names.index("Check for an attempt staged by another run")] + assert "check_uk_candidate.py still-eligible" in duplicate["run"] + # It asks gh whether an earlier attempt's run failed (round 3 of #635). + assert duplicate["env"]["GH_TOKEN"] == "${{ github.token }}" + assert names.index("Install the base environment") < names.index( + "Check for an attempt staged by another run" + ) + own_tree = steps[names.index("Test the candidate commit's own tree")] + assert own_tree["run"].startswith("gh workflow run pr_code_changes.yaml") + assert '--ref "$BRANCH"' in own_tree["run"] + assert own_tree["env"]["GH_TOKEN"] == "${{ github.token }}" + assert names.index("Commit and push the candidate commit") < names.index( + "Test the candidate commit's own tree" + ) + assert job["permissions"]["actions"] == "write" + triggers = yaml.safe_load( + (REPO_ROOT / ".github/workflows/pr_code_changes.yaml").read_text() + )[True] + assert "workflow_dispatch" in triggers + + +def test_still_eligible_reports_a_pair_another_run_is_qualifying(monkeypatch, tmp_path): + from policyengine.provenance.qualification_ledger import attempt_directory + from policyengine.provenance.uk_qualification import stage_attempt + + cli = _cli() + pair = make_pair(model="2.200.0") + candidate = tmp_path / "candidate.json" + candidate.write_text( + json.dumps( + { + "pair": pair.canonical_payload(), + "pair_id": pair.pair_id, + "base_commit": "b" * 40, + "force": False, + } + ) + ) + outputs = [] + monkeypatch.setattr(cli, "set_output", lambda **kwargs: outputs.append(kwargs)) + monkeypatch.setattr(cli, "utc_now", lambda: "2026-10-16T10:00:00Z") + args = argparse.Namespace(candidate=str(candidate), ledger=str(tmp_path)) + assert cli.cmd_still_eligible(args) == 0 + assert outputs[-1] == {"duplicate": "false"} + stage_attempt( + attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12), + {"pair": pair.canonical_payload(), "pair_id": pair.pair_id}, + HARNESS, + owner_run_id="7", + staged_at="2026-10-16T09:30:00Z", + ) + assert cli.cmd_still_eligible(args) == 0 + assert outputs[-1] == {"duplicate": "true"} + # A forced run is never turned into a duplicate (round 3 of #635: the + # recheck dropped force and still passed every test). + payload = json.loads(candidate.read_text()) + candidate.write_text(json.dumps({**payload, "force": True})) + assert cli.cmd_still_eligible(args) == 0 + assert outputs[-1] == {"duplicate": "false"} + + def test_preparation_syncs_the_candidate_lock_before_using_it(): """Snapshots, the bundle check and staging must run in the candidate's environment: synced from the lock preparation regenerates, after it is @@ -154,7 +284,9 @@ def test_head_replacement_and_readiness_share_one_serialization( path = Path(os.environ["PR_STATE"]) state = json.loads(path.read_text()) command, args = Path(sys.argv[0]).name, sys.argv[1:] -if command == "gh" and args[:2] == ["pr", "list"]: +if command == "gh" and args[:1] == ["api"] and "--paginate" in args: + # The REST listing of this repository's pull request to main, through + # the step's jq (only a ready one's number is printed). if state["status"] == "ready": print("574") elif command == "gh" and args[:2] == ["pr", "ready"]: @@ -195,6 +327,7 @@ def prepare(): "BRANCH": branch, "PAIR_ID": qualified.pair_id, "PR_STATE": str(state_path), + "GITHUB_REPOSITORY": "PolicyEngine/policyengine.py", }, check=True, capture_output=True, @@ -216,10 +349,31 @@ def fake_run(argv, **kwargs): state = json.loads(state_path.read_text()) return SimpleNamespace( stdout=json.dumps( - [{"headRefName": branch, "headRefOid": state["head"]}] + [ + { + "number": 1, + "headRefName": branch, + "headRefOid": state["head"], + "isDraft": state["status"] != "ready", + "isCrossRepository": False, + "baseRefName": "main", + } + ] ) ) + # Promotion reads the branch head and checks main is its parent. + if argv[:2] == ["git", "fetch"]: + return SimpleNamespace(returncode=0, stdout="") + if argv[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + head = json.loads(state_path.read_text())["head"] + return SimpleNamespace(returncode=0, stdout=head) + if argv[:2] == ["git", "rev-parse"]: + return SimpleNamespace(returncode=0, stdout="p" * 40) + if argv[:2] == ["git", "ls-remote"]: + return SimpleNamespace(returncode=0, stdout="p" * 40 + "\trefs/heads/main") assert argv[:3] == ["gh", "pr", "ready"] + if "--undo" in argv: + return SimpleNamespace(returncode=0) if not prepare_first: schedule("prepare", prepare) state = json.loads(state_path.read_text()) @@ -274,7 +428,9 @@ def test_catch_up_dispatches_recovery_for_the_original_pair(tmp_path): ) pair = make_pair(model="2.200.0", data="1.58.9") candidate_path = tmp_path / "candidate.json" - candidate_path.write_text(json.dumps({"pair": pair.canonical_payload()})) + candidate_path.write_text( + json.dumps({"pair": pair.canonical_payload(), "pair_id": pair.pair_id}) + ) command_log = tmp_path / "commands.jsonl" binary_dir = tmp_path / "bin" binary_dir.mkdir() @@ -331,6 +487,10 @@ def test_catch_up_dispatches_recovery_for_the_original_pair(tmp_path): "-f", "data_version=1.58.9", "-f", + f"core_version={pair.core.version}", + "-f", + f"pair_id={pair.pair_id}", + "-f", "force=false", ] ] @@ -341,7 +501,7 @@ def test_catch_up_dispatches_recovery_for_the_original_pair(tmp_path): @pytest.mark.parametrize("head_matches", [False, True]) @pytest.mark.parametrize("verdict", ["qualified", "rejected", "accepted_review"]) def test_refresh_checks_reruns_only_failed_pr_runs_for_the_qualified_head( - monkeypatch, tmp_path, head_matches, verdict + monkeypatch, tmp_path, capsys, head_matches, verdict ): cli = _cli() receipt = make_receipt( @@ -362,42 +522,275 @@ def test_refresh_checks_reruns_only_failed_pr_runs_for_the_qualified_head( branch = candidate_branch(receipt.pair) head = receipt.harness.git_commit if head_matches else "f" * 40 monkeypatch.setattr(cli, "latest_receipts", lambda root, **_: [receipt]) - monkeypatch.setattr(cli, "_open_candidate_branches", lambda: {branch: head}) + monkeypatch.setattr( + cli, "_candidate_pull_requests", lambda: [_candidate_pr(1, branch, head)] + ) + monkeypatch.setattr(cli, "_now", lambda: _instant(NOW)) + code, docs, guard, old = ( + ".github/workflows/pr_code_changes.yaml", + ".github/workflows/pr_docs.yaml", + ".github/workflows/guard.yaml", + ".github/workflows/old.yaml", + ) + fork = {"head_repository": "someone/policyengine.py"} + other = {"head_branch": "candidate/uk/other"} runs = [ - { - "id": identifier, - "head_sha": sha, - "event": event, - "status": status, - "conclusion": conclusion, - } - for identifier, sha, event, status, conclusion in ( - (11, head, "pull_request", "completed", "failure"), - (12, head, "pull_request", "completed", "success"), - (13, head, "pull_request", "in_progress", "failure"), - (14, "a" * 40, "pull_request", "completed", "failure"), - (15, head, "push", "completed", "failure"), - (16, head, "pull_request", "completed", "timed_out"), + _listed_run(identifier, sha, branch, path, event, status, conclusion, attempt) + | extra + for identifier, sha, path, event, status, conclusion, attempt, extra in ( + # Replaced by run 16 of the same workflow and event: left alone, + # so its checks cannot come back newer (round 3 of #635). + (11, head, code, "pull_request", "completed", "failure", 1, {}), + (12, head, docs, "pull_request", "completed", "success", 1, {}), + (13, head, guard, "pull_request", "in_progress", None, 1, {}), + (14, "a" * 40, code, "pull_request", "completed", "failure", 1, {}), + (15, head, code, "push", "completed", "failure", 1, {}), + # GitHub refuses this re-run: refresh warns and the rest still go + # (round 4; round 5 lists it first). + (16, head, code, "pull_request", "completed", "timed_out", 1, {}), + # The run on the candidate's own tree that promotion requires; a + # job timeout cancels the run (round 3 of #635). + (17, head, code, "workflow_dispatch", "completed", "cancelled", 1, {}), + # Already run as often as the cap allows (four attempts). + (18, head, docs, "workflow_dispatch", "completed", "failure", 4, {}), + (19, head, guard, "workflow_dispatch", "completed", "failure", 3, {}), + # Round 5 of #635: a fork's newer run at the same commit, and one + # of another branch here, are not the candidate's: neither is + # re-run, nor replaces the candidate's own runs 16 and 12. + (21, head, code, "pull_request", "completed", "failure", 1, fork), + (22, head, docs, "pull_request", "completed", "failure", 1, other), + # Round 5: GitHub re-runs only within 30 days of the first attempt, + # and refresh re-runs only failures, timeouts and cancellations. + ( + 23, + head, + old, + "workflow_dispatch", + "completed", + "failure", + 1, + {"created_at": OLD}, + ), + (24, head, old, "pull_request", "completed", "action_required", 1, {}), ) ] calls = [] def fake_run(argv, **kwargs): calls.append(argv) + if argv[:3] == ["gh", "api", "graphql"]: + return SimpleNamespace( + returncode=0, stdout=graphql_base_changes_output(argv, set()) + ) + if argv[:2] == ["gh", "api"] and argv[4:5] == PULLS: + assert f"head=PolicyEngine:{branch}" in argv + return SimpleNamespace( + returncode=0, + stdout=rest_pulls_output(argv, [_candidate_pr(1, branch, head)]), + ) if argv[:2] == ["gh", "api"]: assert "--paginate" in argv assert f"head_sha={receipt.harness.git_commit}" in argv[2] - assert "event=pull_request" in argv[2] - return SimpleNamespace(stdout="\n".join(json.dumps(run) for run in runs)) - assert argv[:3] == ["gh", "run", "rerun"] - assert "--failed" in argv and kwargs["check"] is True - return SimpleNamespace(returncode=0) + return SimpleNamespace( + returncode=0, stdout="\n".join(json.dumps(run) for run in runs) + ) + assert argv[:3] == ["gh", "run", "rerun"] and kwargs["check"] is False + return SimpleNamespace(returncode=1 if argv[3] == "16" else 0) monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=fake_run)) args = cli.parser().parse_args(["refresh-pr-checks", "--ledger", str(tmp_path)]) assert args.func(args) == 0 - reruns = [call[3] for call in calls if call[:3] == ["gh", "run", "rerun"]] + reruns = [ + (call[3], "--failed" in call) + for call in calls + if call[:3] == ["gh", "run", "rerun"] + ] + output = capsys.readouterr().out if receipt.promotable and head_matches: - assert reruns == ["11", "16"] + # Failed jobs only for failed and timed-out runs; the cancelled one in + # full; the one at the cap, the old one and the fork's not at all. + assert reruns == [("16", True), ("17", False), ("19", True)] + assert "::warning::GitHub did not re-run timed_out run 16" in output else: assert calls == [] + + +NOW = "2026-10-16T12:00:00Z" +OLD = "2026-09-15T11:00:00Z" + + +def _candidate_pr(number, branch, head, **fields): + """A same-repository pull request from ``branch`` to main, open, as + `gh pr list` reads it.""" + + return { + "number": number, + "baseRefName": "main", + "headRefName": branch, + "headRefOid": head, + "isDraft": True, + "isCrossRepository": False, + "state": "OPEN", + **fields, + } + + +def _instant(text): + import datetime + + return datetime.datetime.fromisoformat(text.replace("Z", "+00:00")) + + +def _listed_run(identifier, sha, branch, path, event, status, conclusion, attempt): + """A run as `_commit_runs` reads it: the candidate's own by default.""" + + return { + "id": identifier, + "head_sha": sha, + "head_branch": branch, + "head_repository": "PolicyEngine/policyengine.py", + "path": path, + "event": event, + "status": status, + "conclusion": conclusion, + "check_suite_id": 1000 + identifier, + "run_attempt": attempt, + "created_at": "2026-10-16T06:00:00Z", + } + + +def test_refresh_reruns_each_candidate_only_its_own_runs(monkeypatch, tmp_path, capsys): + """Round 5 of #635: two candidates qualified at the same commit share its + runs listing; each re-runs only its own branch's runs, and one whose runs + cannot be read is reported while the other is still refreshed.""" + + cli = _cli() + first = make_receipt(pair=make_pair(model="2.200.0")) + second = make_receipt(pair=make_pair(model="2.201.0")) + head = first.harness.git_commit + assert second.harness.git_commit == head + branches = {candidate_branch(r.pair): head for r in (first, second)} + pull_requests = [ + _candidate_pr(number, name, head) + for number, name in enumerate(branches, start=1) + ] + monkeypatch.setattr(cli, "latest_receipts", lambda root, **_: [first, second]) + monkeypatch.setattr(cli, "_candidate_pull_requests", lambda: pull_requests) + monkeypatch.setattr(cli, "_now", lambda: _instant(NOW)) + code = ".github/workflows/pr_code_changes.yaml" + first_branch, second_branch = branches + runs = [ + _listed_run( + 31, head, first_branch, code, "pull_request", "completed", "failure", 1 + ), + _listed_run( + 32, head, second_branch, code, "pull_request", "completed", "failure", 1 + ), + ] + calls = [] + unreadable = {"once": True} + + def fake_run(argv, **kwargs): + calls.append(argv) + if argv[:3] == ["gh", "api", "graphql"]: + return SimpleNamespace( + returncode=0, stdout=graphql_base_changes_output(argv, set()) + ) + if argv[:2] == ["gh", "api"] and argv[4:5] == PULLS: + return SimpleNamespace( + returncode=0, stdout=rest_pulls_output(argv, pull_requests) + ) + if argv[:2] == ["gh", "api"]: + if unreadable.pop("once", False): + return SimpleNamespace(returncode=1, stdout="") + return SimpleNamespace( + returncode=0, stdout="\n".join(json.dumps(run) for run in runs) + ) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=fake_run)) + args = cli.parser().parse_args(["refresh-pr-checks", "--ledger", str(tmp_path)]) + assert args.func(args) == 0 + output = capsys.readouterr().out + assert f"::warning::could not read the runs of {first_branch}" in output + assert [call[3] for call in calls if call[:3] == ["gh", "run", "rerun"]] == ["32"] + + +@pytest.mark.parametrize( + "shared", ["another base", "a base change", "unreadable", "unreadable history"] +) +def test_refresh_leaves_a_candidate_whose_branch_runs_are_shared( + monkeypatch, tmp_path, capsys, shared +): + """Round 6 of #635: a pull request from the candidate branch to another + base runs on the same commit and branch, so the candidate's runs cannot + be told from its runs; refresh re-runs none of them (promotion refuses + the candidate anyway), and the next candidate is still refreshed. Round + 7: so does the candidate's own pull request once its base was changed. + The same holds while that branch's pull requests, or their base history, + cannot be read.""" + + cli = _cli() + first = make_receipt(pair=make_pair(model="2.200.0")) + second = make_receipt(pair=make_pair(model="2.201.0")) + head = first.harness.git_commit + first_branch, second_branch = (candidate_branch(r.pair) for r in (first, second)) + pull_requests = [ + _candidate_pr(1, first_branch, head), + _candidate_pr(2, second_branch, head), + ] + other = _candidate_pr(7, first_branch, head, baseRefName="release") + monkeypatch.setattr(cli, "latest_receipts", lambda root, **_: [first, second]) + monkeypatch.setattr(cli, "_candidate_pull_requests", lambda: pull_requests) + monkeypatch.setattr(cli, "_now", lambda: _instant(NOW)) + code = ".github/workflows/pr_code_changes.yaml" + runs = [ + _listed_run( + 31, head, first_branch, code, "pull_request", "completed", "failure", 1 + ), + _listed_run( + 32, head, second_branch, code, "pull_request", "completed", "failure", 1 + ), + ] + calls = [] + + def fake_run(argv, **kwargs): + calls.append(argv) + if argv[:3] == ["gh", "api", "graphql"]: + number = int(next(v for v in argv if v.startswith("number="))[7:]) + if number == 1 and shared == "unreadable history": + return SimpleNamespace(returncode=1, stdout="") + changed = {1} if shared == "a base change" else set() + return SimpleNamespace( + returncode=0, stdout=graphql_base_changes_output(argv, changed) + ) + if argv[:2] == ["gh", "api"] and argv[4:5] == PULLS: + if f"head=PolicyEngine:{first_branch}" in argv and shared == "unreadable": + return SimpleNamespace(returncode=1, stdout="") + listing = [*pull_requests, other] if shared == "another base" else [] + return SimpleNamespace( + returncode=0, + stdout=rest_pulls_output(argv, listing or pull_requests), + ) + if argv[:2] == ["gh", "api"]: + return SimpleNamespace( + returncode=0, stdout="\n".join(json.dumps(run) for run in runs) + ) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(cli, "subprocess", SimpleNamespace(run=fake_run)) + args = cli.parser().parse_args(["refresh-pr-checks", "--ledger", str(tmp_path)]) + assert args.func(args) == 0 + output = capsys.readouterr().out + if shared.startswith("unreadable"): + assert f"::warning::could not read the pull requests from {first_branch}" in ( + output + ) + else: + assert f"::warning::not re-running {first_branch}'s runs" in output + assert ( + "#7 (to release, open)" + if shared == "another base" + else "#1 (the candidate's own, after a base change)" + ) in output + assert [call[3] for call in calls if call[:3] == ["gh", "run", "rerun"]] == ["32"]