Compare each pull request's calibration fit with its base build's

.github/workflows/pull_request.yaml

@@ -72,6 +72,10 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -86,21 +90,161 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      - name: Save calibration log (constituencies)
-        uses: actions/upload-artifact@v7
-        with:
-          name: constituency_calibration_log.csv
-          path: constituency_calibration_log.csv
-
-      - name: Save calibration log (local authorities)
-        uses: actions/upload-artifact@v7
-        with:
-          name: la_calibration_log.csv
-          path: la_calibration_log.csv
-
       - name: Run tests
         run: uv run --frozen make test
         env:
           # Relaxes built-dataset outcome tolerances to match the reduced
           # fidelity of the TESTING build above (see test_cgt_band_donors).
           TESTING: "1"
+      # After the tests, whatever they did. The raw calibration logs stay on
+      # the runner: a target supported by a few households, divided by a
+      # published count, could give one household's value back. Only the
+      # summary the fit comparison needs is uploaded (utils/fit_report.py).
+      - name: Summarise the calibration fit
+        if: ${{ !cancelled() }}
+        run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json
+      - name: Save the fit summary
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: fit_summary.json
+          path: fit_summary.json
+  # The no-worsening check (W3). The base is built the same way as the head:
+  # TESTING, one clone and the head's lockfile, so both builds run one model
+  # and environment and the comparison measures the data change. As in the
+  # test job, only the fit summary leaves it. No dataset or calibration log
+  # is uploaded, and nothing is cached: setup-uv restores uv's package cache
+  # and saves none.
+  base-build:
+    name: Build the base for comparison
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+      PYTHONUNBUFFERED: "1"
+      PE_UK_DATA_OA_CLONES: "1"
+    steps:
+      - name: Check out the head, for its lockfile
+        uses: actions/checkout@v6
+        with:
+          path: head
+      - name: Check out the base
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+      - name: Sync the head's locked environment for the base
+        run: |
+          cp head/uv.lock base/uv.lock
+          cd base && uv sync --frozen --all-extras
+      - name: Download data inputs
+        working-directory: base
+        run: uv run --frozen make download
+      - name: Build the base datasets
+        working-directory: base
+        run: uv run --frozen make data
+        env:
+          TESTING: "1"
+      - name: Summarise the base's calibration fit
+        run: |
+          cp head/policyengine_uk_data/utils/fit_report.py base/policyengine_uk_data/utils/fit_report.py
+          uv run --frozen --project base python -m policyengine_uk_data.utils.fit_report summarise base/constituency_calibration_log.csv --out base_fit_summary.json
+      - name: Save the base fit summary
+        uses: actions/upload-artifact@v7
+        with:
+          name: base_fit_summary.json
+          path: base_fit_summary.json
+  fit-comparison:
+    name: Calibration fit against the base
+    needs: [test, base-build]
+    # Report a failed side too: an earlier passing comment must not stand.
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+      - name: Sync locked environment
+        run: uv sync --frozen
+      - name: Download the fit summaries
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          pattern: "*fit_summary.json"
+          path: logs
+          merge-multiple: true
+      - name: Compare the final fits
+        id: compare
+        env:
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          marker='<!-- policyengine-uk-data-fit-comparison -->'
+          if [ ! -f logs/fit_summary.json ] || [ ! -f logs/base_fit_summary.json ]; then
+            {
+              echo "$marker"
+              echo '## Calibration fit against the base build: FAIL'
+              echo
+              echo "One of the builds produced no fit summary, so the fit could not be compared. See the [workflow run]($RUN_URL)."
+            } > fit.md
+            echo "status=1" >> "$GITHUB_OUTPUT"
+          else
+            set +e
+            uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \
+              logs/base_fit_summary.json logs/fit_summary.json --markdown body.md
+            echo "status=$?" >> "$GITHUB_OUTPUT"
+            set -e
+            { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md
+          fi
+          cat fit.md >> "$GITHUB_STEP_SUMMARY"
+      - name: Post or update the pull request comment
+        if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          marker='<!-- policyengine-uk-data-fit-comparison -->'
+          if [ ! -s fit.md ]; then
+            # The job failed before the comparison could write a comment.
+            {
+              echo "$marker"
+              echo '## Calibration fit against the base build: FAIL'
+              echo
+              echo "The comparison did not run. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+            } > fit.md
+          fi
+          existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
+            --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \
+            | tail -n 1)
+          if [ -n "$existing" ]; then
+            gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@fit.md > /dev/null
+          else
+            gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@fit.md > /dev/null
+          fi
+      - name: Fail on a fit regression
+        if: steps.compare.outputs.status != '0'
+        run: |
+          echo "::error::The head build's calibration fit is worse than the base's; see the job summary."
+          exit 1

changelog.d/fit-comparison.added.md

@@ -0,0 +1 @@
+Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets in a worse error band on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only each target's error band (within 10%, 10-25%, 25-50%, 50-100%, over 100%) is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).

policyengine_uk_data/targets/fit_policy.yaml

@@ -0,0 +1,47 @@
+# When a pull request's build fails or flags against its base build's fit
+# (utils/fit_report.py). Both builds run the same way in pull request CI:
+# TESTING, 32 epochs, one output-area clone, the head's lockfile.
+#
+# The share allowance was adopted under cos d1344 (15 October 2026), from the final-epoch fits of
+# five builds of the same data code: docs, changelog and CI pull requests
+# while main's data code was unchanged from b34a4ea2 to 2b9c1c2c (runs
+# 38006434333, 37959581668, 38009046475, 37960852139, 37864653693).
+schema_version: 1
+
+share_within_10pct:
+  # Fail when the share of national, or of local, targets within 10% of
+  # their values falls by more than this. Across the 20 ordered pairs of the
+  # five builds the national share moved by at most 0.93pp and the local by
+  # at most 0.69pp, so 2pp is about twice the same-code spread.
+  # Alternatives: 1pp leaves no margin over that spread while #598 (builds
+  # of the same code fall into two weight sets) is unexplained. 5pp would
+  # pass the 2.4pp local drop of #551's build against the build of the
+  # 1.58.1 release branch (runs 38661397028 and 38228500493).
+  allowed_drop: 0.02
+
+national_target:
+  # Flag, for review, each national target whose error band is worse on the
+  # head than on the base. Bands of |estimate / target - 1|: within 10%,
+  # 10-25%, 25-50%, 50-100%, over 100% (fit_report.BANDS). Across the 20
+  # ordered pairs of the five same-code builds above, 2 to 12 of the 644
+  # national targets move to a worse band per pair (mean 7), so a list stays
+  # short. #551's build moves 31 and #579's 27 against the 1.58.1 release
+  # branch's build.
+  #
+  # This replaces flagging an error rise of more than 0.2 (adopted under
+  # d1344 on 15 October), which needed each target's error in the uploaded
+  # summary. Sol's re-review of #599 found no safe way to publish them: the
+  # dataset repeats households (output-area clones, SPI and CGT donors), so
+  # a count of rows can't show how many respondents support an estimate, and
+  # one household can dominate a target. A band is a bound, not a value.
+  # The old rule gave 0 to 6 flags per same-code pair. Bands still can't see
+  # a move inside one band (say 30% to 45%). The replacement is a W3
+  # threshold call under d1344's delegation (thresholds on evidence, with an
+  # independent review agreeing), pending that review. Per-target moves
+  # don't fail until #598 is fixed.
+  #
+  # Run ids of the five builds and their saved log folders: 38006434333
+  # (docs-ebdbbce5), 37959581668 (docs-2a071400), 38009046475
+  # (changelog-6af935fb), 37960852139 (changelog-bb41c3ca), 37864653693
+  # (ci-push-gate-03b3ecd9); the 1.58.1 release branch is 38228500493.
+  flag: worse_band