Turn on automatic UK promotion under d1344

.github/workflows/certify-uk-release.yaml

@@ -7,8 +7,10 @@
 # mandatory gates on that exact commit, replayed by policyengine-scorecard,
 # and recorded in the uk-qualification-ledger branch. Promotion is
 # serialized: it readies the newest promotable candidate PR that advances
-# main and closes superseded ones. Merging stays a reviewed step until
-# cos d1344 rules on automatic promotion.
+# main, closes superseded ones, and (cos d1344) fast-forwards main to its
+# qualified commit once every check on it has passed, which succeeds only
+# while main is still the commit it was built on; otherwise the catch-up
+# requalifies it on current main.
 #
 # Triggers: a policyengine-uk publish or uk-data release (repository
 # dispatch), a two-hourly catch-up, or a manual run. A manual run with
@@ -29,6 +31,12 @@ on:
       data_version:
         description: uk-data release (default newest)
         required: false
+      core_version:
+        description: policyengine-core version (default the certified one if the model accepts it)
+        required: false
+      pair_id:
+        description: The exact pair to qualify; the run fails if the versions resolve to another
+        required: false
       force:
         description: Qualify even if the ledger already has a terminal receipt
         type: boolean
@@ -104,6 +112,8 @@ jobs:
           PAYLOAD_VERSION: ${{ github.event.client_payload.version }}
           INPUT_MODEL: ${{ inputs.model_version }}
           INPUT_DATA: ${{ inputs.data_version }}
+          INPUT_CORE: ${{ inputs.core_version }}
+          INPUT_PAIR_ID: ${{ inputs.pair_id }}
           INPUT_FORCE: ${{ inputs.force }}
         run: |
           model="${INPUT_MODEL:-}"
@@ -116,6 +126,8 @@ jobs:
           args=(resolve --out candidate.json --ledger ledger)
           [[ -n "$model" ]] && args+=(--model-version "$model")
           [[ -n "$data" ]] && args+=(--data-version "$data")
+          [[ -n "${INPUT_CORE:-}" ]] && args+=(--core-version "$INPUT_CORE")
+          [[ -n "${INPUT_PAIR_ID:-}" ]] && args+=(--pair-id "$INPUT_PAIR_ID")
           [[ "${INPUT_FORCE:-false}" == "true" ]] && args+=(--force)
           python scripts/check_uk_candidate.py "${args[@]}"
       - uses: actions/upload-artifact@v4
@@ -133,6 +145,10 @@ jobs:
       # promotion can ready a replacement after checking the previous head.
       group: promote-uk
       cancel-in-progress: false
+    permissions:
+      contents: write
+      pull-requests: write
+      actions: write
     outputs:
       candidate_commit: ${{ steps.stage.outputs.candidate_commit }}
       duplicate: ${{ steps.duplicate.outputs.duplicate }}
@@ -152,37 +168,39 @@ jobs:
           ref: ${{ needs.resolve.outputs.base_commit }}
           token: ${{ steps.app-token.outputs.token }}
           fetch-depth: 0
-      # Another run may have staged this attempt while this one waited for
-      # the promotion concurrency group; never overwrite its evidence.
-      - name: Check for an attempt staged by another run
-        id: duplicate
-        env:
-          ATTEMPT: ledger/uk/${{ needs.resolve.outputs.pair_id }}/attempts/${{ needs.resolve.outputs.attempt_key }}
-        run: |
-          .github/qualification-ledger.sh checkout ledger
-          if [[ -d "$ATTEMPT" ]]; then
-            echo "duplicate=true" >> "$GITHUB_OUTPUT"
-            echo "Attempt already staged by another run; nothing to do."
-          else
-            echo "duplicate=false" >> "$GITHUB_OUTPUT"
-          fi
-          git worktree remove --force ledger
       - uses: actions/download-artifact@v4
-        if: steps.duplicate.outputs.duplicate == 'false'
         with:
           name: candidate
           path: ${{ runner.temp }}/candidate
       - uses: astral-sh/setup-uv@v8.1.0
-        if: steps.duplicate.outputs.duplicate == 'false'
         with:
           version: ${{ env.UV_VERSION }}
       - uses: actions/setup-python@v6
-        if: steps.duplicate.outputs.duplicate == 'false'
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install the base environment
-        if: steps.duplicate.outputs.duplicate == 'false'
         run: uv sync --frozen --extra uk --extra dev --python ${{ env.PYTHON_VERSION }}
+      # Another run may have staged this attempt, or another attempt for the
+      # pair from another main commit, while this one waited for the
+      # promotion concurrency group: recheck against the ledger as it is now,
+      # and never overwrite another run's evidence.
+      - name: Check for an attempt staged by another run
+        id: duplicate
+        env:
+          ATTEMPT: ledger/uk/${{ needs.resolve.outputs.pair_id }}/attempts/${{ needs.resolve.outputs.attempt_key }}
+          # still-eligible asks gh whether an earlier attempt's run failed;
+          # without a token every unexpired failed run would read as live.
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          .github/qualification-ledger.sh checkout ledger
+          if [[ -d "$ATTEMPT" ]]; then
+            echo "duplicate=true" >> "$GITHUB_OUTPUT"
+            echo "Attempt already staged by another run; nothing to do."
+          else
+            uv run --no-sync python scripts/check_uk_candidate.py still-eligible \
+              --candidate "$RUNNER_TEMP/candidate/candidate.json" --ledger ledger
+          fi
+          git worktree remove --force ledger
       - name: Certify the pair and regenerate pins, lock, TROs and constants
         if: steps.duplicate.outputs.duplicate == 'false'
         env:
@@ -216,8 +234,12 @@ jobs:
         run: |
           # A forced attempt may replace an already qualified head. Revoke
           # its ready status before exposing any new, unqualified commit.
-          ready_pr=$(gh pr list --head "$BRANCH" --state open --json number,isDraft \
-            --jq '.[] | select(.isDraft == false) | .number')
+          # Only this repository's pull request to main is the candidate's:
+          # head=<owner>:<branch> lists only this repository's, so a fork's
+          # with the same branch name can neither shadow nor crowd it out.
+          ready_pr=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \
+            -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \
+            --paginate --jq '.[] | select(.draft == false) | .number')
           if [[ -n "$ready_pr" ]]; then
             gh pr ready "$ready_pr" --undo
           fi
@@ -226,6 +248,15 @@ jobs:
               -c user.email="qualification@policyengine.org" \
               commit -q -m "Certify UK candidate pair ${PAIR_ID:0:12} for qualification"
           git push --force origin "HEAD:refs/heads/$BRANCH"
+      # Pull request runs test the candidate merged with main as it was then.
+      # Promotion needs the commit's own tree tested, so run the code-changes
+      # workflow on the candidate branch itself.
+      - name: Test the candidate commit's own tree
+        if: steps.duplicate.outputs.duplicate == 'false'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BRANCH: ${{ needs.resolve.outputs.branch }}
+        run: gh workflow run pr_code_changes.yaml --repo "$GITHUB_REPOSITORY" --ref "$BRANCH"
       - name: Stage the attempt in the ledger
         id: stage
         if: steps.duplicate.outputs.duplicate == 'false'
@@ -248,8 +279,13 @@ jobs:
           Ledger: https://github.com/${GITHUB_REPOSITORY}/tree/${LEDGER_BRANCH}/uk/${PAIR_ID}
 
           🤖 Generated with [Claude Code](https://claude.com/claude-code)"
-          if gh pr view "$BRANCH" --json number >/dev/null 2>&1; then
-            gh pr edit "$BRANCH" --body "$body"
+          # By number: gh looks a branch name up among pull requests a fork's
+          # can shadow. head=<owner>:<branch> lists only this repository's.
+          number=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \
+            -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \
+            --paginate --jq '.[].number')
+          if [[ -n "$number" ]]; then
+            gh pr edit "$number" --body "$body"
           else
             gh pr create --draft --base main --head "$BRANCH" --title "$title" --body "$body"
           fi
@@ -364,7 +400,15 @@ jobs:
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           BRANCH: ${{ needs.resolve.outputs.branch }}
-        run: gh pr edit "$BRANCH" --body-file "$RUNNER_TEMP/pr-body.md"
+        run: |
+          number=$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \
+            -f head="${GITHUB_REPOSITORY%%/*}:$BRANCH" -f base=main -f state=open \
+            --paginate --jq '.[].number')
+          if [[ -z "$number" ]]; then
+            echo "::error::no open pull request from $BRANCH to main"
+            exit 1
+          fi
+          gh pr edit "$number" --body-file "$RUNNER_TEMP/pr-body.md"
 
   catch-up:
     if: github.event_name == 'schedule'
@@ -400,13 +444,16 @@ jobs:
           from pathlib import Path
 
           for line in Path("recoverable.jsonl").read_text().splitlines():
-              pair = json.loads(line)["pair"]
+              entry = json.loads(line)
+              pair, pair_id = entry["pair"], entry["pair_id"]
               subprocess.run(
                   [
                       "gh", "workflow", "run", "certify-uk-release.yaml",
                       "--repo", os.environ["GITHUB_REPOSITORY"], "--ref", "main",
                       "-f", f"model_version={pair['model']['version']}",
                       "-f", f"data_version={pair['data']['release_version']}",
+                      "-f", f"core_version={pair['core']['version']}",
+                      "-f", f"pair_id={pair_id}",
                       "-f", "force=false",
                   ],
                   check=True,
@@ -427,6 +474,36 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: python scripts/check_uk_candidate.py refresh-pr-checks --ledger ledger
+      # Automatic promotion needs the target built on current main and compared
+      # against main's certificate. A run without force requalifies it once per
+      # main commit, one at a time; repeated catch-ups are duplicates.
+      - name: Requalify a promotable target qualified against an earlier certificate
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/check_uk_candidate.py requalify --ledger ledger > requalify.jsonl
+          python - <<'PY'
+          import json
+          import os
+          import subprocess
+          from pathlib import Path
+
+          for line in Path("requalify.jsonl").read_text().splitlines():
+              entry = json.loads(line)
+              pair, pair_id = entry["pair"], entry["pair_id"]
+              subprocess.run(
+                  [
+                      "gh", "workflow", "run", "certify-uk-release.yaml",
+                      "--repo", os.environ["GITHUB_REPOSITORY"], "--ref", "main",
+                      "-f", f"model_version={pair['model']['version']}",
+                      "-f", f"data_version={pair['data']['release_version']}",
+                      "-f", f"core_version={pair['core']['version']}",
+                      "-f", f"pair_id={pair_id}",
+                      "-f", "force=false",
+                  ],
+                  check=True,
+              )
+          PY
 
   accept:
     if: github.event_name == 'workflow_dispatch' && inputs.accept_pair_id != ''
@@ -475,10 +552,19 @@ jobs:
       group: promote-uk
       cancel-in-progress: false
     steps:
+      # Promotion pushes main as the App, so the push starts main's release
+      # workflow; a push with the job's own token would not.
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
       - uses: actions/checkout@v6
         with:
           ref: main
           fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
       - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: ${{ env.UV_VERSION }}
@@ -487,12 +573,6 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install policyengine without country models
         run: uv pip install --system -e .
-      - name: Generate GitHub App token
-        id: app-token
-        uses: actions/create-github-app-token@v1
-        with:
-          app-id: ${{ secrets.APP_ID }}
-          private-key: ${{ secrets.APP_PRIVATE_KEY }}
       - name: Promote the newest qualified pair that advances main
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}

changelog.d/634.changed.md

@@ -0,0 +1 @@
+The UK qualification replay gate applies policy d1344-replay-regressions-v1. It fails lost results, lost responses (a measure-year the candidate lists as inert, or a measure-year total of £0 against a base at least the diff's bound away), mapping flags and incomplete grids. It sends every other flagged move to a named reviewer. Each receipt records the policy id and its text's SHA-256. Automatic UK promotion is on (d1344): a qualified candidate still built on current main is moved onto main, by a push leased on its parent, once every check of its own runs on its own tree has passed. Runs of pull requests from other branches or forks at the same commit are not counted; a candidate whose runs may not all be its own pull request's against main (its branch has another pull request that is open or ran at its commit, whatever that one's base, or its own pull request's base was ever changed) is refused with an error, since GitHub does not record which pull request or base a run is for, and a base renamed into main records no event. Candidate pull requests are found through GitHub's listing of this repository's own branches, which forks' pull requests cannot crowd out, and every write to one goes by its number. Refresh re-runs a candidate's failed, timed-out or cancelled runs up to four attempts within GitHub's 30-day window, and a candidate left waiting on a run it will not re-run fails the promote run with an error naming it.

docs/engineering/runbooks/uk-qualification.md

@@ -60,3 +60,61 @@ some from fewer than 10 records, so never commit, paste or publish it.
 
 4. Read `failing[...]["detail"]` locally. Report only the finding (which
    condition failed, on which route), never the figures.
+
+## A candidate stranded before automatic promotion
+
+Automatic promotion moves main to a qualified candidate only once the newest
+run of each workflow and event on the candidate branch, from this repository,
+has passed, including a `pr_code_changes.yaml` run dispatched on the branch
+itself. Runs of pull requests from other branches or forks at the same
+commit do not count. The `refresh-pr-checks` step re-runs a failed, timed-out or
+cancelled run until it has had four attempts, within GitHub's 30 days of its
+first attempt.
+
+When the candidate waits on a run that refresh will not re-run, the promote
+job fails with `::error::<branch> waits on runs refresh will not re-run`,
+naming each run, its result and its attempts. That covers a run at four
+attempts, one older than 30 days, and one whose result refresh never re-runs
+(`action_required`, `skipped`, `stale`, `startup_failure` and the like). The
+job fails on every scheduled run until one of these clears it:
+
+- Fix the cause, then re-run the named run by hand
+  (`gh run rerun <run id> --repo PolicyEngine/policyengine.py`); a passing
+  newest attempt clears it. GitHub refuses this for a run whose first
+  attempt is more than 30 days old.
+- For the run on the candidate's own tree, dispatch a new one:
+  `gh workflow run pr_code_changes.yaml --repo PolicyEngine/policyengine.py --ref <branch>`.
+- Requalify the pair: run `certify-uk-release` with its versions, `pair_id`
+  and `force` set.
+
+### Another pull request from the candidate branch
+
+Any other pull request from a candidate branch runs the same workflows with
+the same commit and branch as the candidate's own pull request, and
+GitHub's run records say neither which pull request a run is for nor which
+base it tested; a pull request's current base does not settle it, since a
+base renamed into main records no event. So while another pull request from
+the branch is open, or after one was closed or merged at the candidate's
+commit (whatever its base), or when the candidate's own pull request's base
+was ever changed, the promote job fails with
+`::error::<branch> is not promoted: runs at its commit may have tested another base`,
+naming the pull requests, and refresh re-runs none of the candidate's runs.
+Close those pull requests, then requalify the pair as above: the new
+candidate commit leaves their runs behind, and prepare opens a new pull
+request to main if none is open.
+
+### What promotion trusts
+
+Promotion guards against mistakes by people who can write to this
+repository and against anything people who cannot can do (forks' pull
+requests, whatever their branch names, and other pull requests opened from
+a candidate branch, which make promotion refuse; one to main that prepare's
+lookup adopts is the candidate's own, and promotion refuses it if its base
+was ever changed). Whatever else happens, main only
+ever becomes the exact qualified commit, built on main as it is at the push,
+after the dispatched `pr_code_changes.yaml` run on that commit's own tree
+passed. It does not try to resist deliberate disguise by people who can
+write here, such as renaming another branch into main to make a run against
+another base look like the candidate's: main is not protected, so they
+could push it directly. Treat any such change as an incident, not as
+something the gate will catch.