Skip to content
Pull request

[DO NOT MERGE] Batch-3 fit attribution: seeded release without #589 #1201

Sign in to view logs

[DO NOT MERGE] Batch-3 fit attribution: seeded release without #589

[DO NOT MERGE] Batch-3 fit attribution: seeded release without #589 #1201

Workflow file for this run

.github/workflows/pull_request.yaml at 5a9da6b
# This workflow is used to check various parts of the
# run prior to merging:
# - The code's formatting
# - Versioning - note that this checks, but does not update
# - That the code builds successfully
# - That any automated tests pass
name: Pull request
on:
pull_request:
branches: [main]
# A new push to a pull request supersedes the run on its previous head. Cancel
# that run instead of letting its ~40-minute dataset build and test job finish
# on a stale commit while it holds one of the org's shared runners.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 10
name: Lint
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.13
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install "ruff>=0.9.0"
- name: Check formatting
run: ruff format --check .
bundle-release-manifest-contract:
name: Validate bundle release manifest contract
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.14
- name: Install uv
uses: astral-sh/setup-uv@v8.1.0
- name: Install package
run: uv pip install --system .
- name: Install bundle validation tooling
# Pin the test-only bundle contract dependency until policyengine-bundles
# has published releases suitable for ordinary dependency specifiers.
run: uv pip install --system pytest "policyengine-bundles @ git+https://github.com/PolicyEngine/policyengine-bundles@8ae9f56fefcf89f69b8a7e3bc49928509c6207be"
- name: Validate release manifest contract
run: python -m pytest policyengine_uk_data/tests/test_release_manifest.py::test_build_release_manifest_validates_against_bundle_contract
test:
name: Test
runs-on: ubuntu-latest
# Slowest observed 2026-10-06..08: 47 minutes (TESTING build plus tests).
# Fail a hung build instead of holding a runner for the 6-hour default.
timeout-minutes: 90
env:
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
PYTHONUNBUFFERED: "1"
PE_UK_DATA_OA_CLONES: "1"
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Install uv
uses: astral-sh/setup-uv@v8.1.0
# Restore uv's package cache but never save one from a job that
# holds the data: its location can be moved (UV_CACHE_DIR).
with:
save-cache: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.13
- name: Sync locked environment
run: uv sync --frozen --all-extras
- name: Download data inputs
run: uv run --frozen make download
env:
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
- name: Build datasets
run: uv run --frozen make data
env:
TESTING: "1"
# The raw calibration logs stay on the runner. They hold every target's
# weighted estimate at every logged epoch with no small-cell control, so a
# target supported by a few households, divided by a published count,
# could give one household's value back, and this repository's
# artifacts are public.
- name: Run tests
if: false # DIAGNOSTIC ONLY (batch-3 attribution): skip tests to save runner time
run: uv run --frozen make test
env:
# Relaxes built-dataset outcome tolerances to match the reduced
# fidelity of the TESTING build above (see test_cgt_band_donors).
TESTING: "1"
# After the tests, whatever they did. The raw calibration logs stay on
# the runner: a target supported by a few households, divided by a
# published count, could give one household's value back. Only the
# summary the fit comparison needs is uploaded (utils/fit_report.py).
- name: Summarise the calibration fit
if: ${{ !cancelled() }}
run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json
- name: Save the fit summary
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: fit_summary.json
path: fit_summary.json
# After the tests, and whatever they did: the measurement downloads the
# ONS projection, and an ONS outage must not skip the test suite.
- name: Measure the income distribution and single ages
if: ${{ !cancelled() }}
# A failed measurement fails the comparison (missing report), not
# the test job, which stays about the tests.
continue-on-error: true
run: uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out distribution_report.json
- name: Save the distribution report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: distribution_report.json
path: distribution_report.json
- name: Check out the aggregate-impact tool
if: ${{ !cancelled() }}
# A missing result fails the comparison, not the test job.
continue-on-error: true
uses: actions/checkout@v6
with:
repository: PolicyEngine/policyengine-uk
ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
path: pe-uk-tool
sparse-checkout: |
policyengine_uk/utils/aggregate_impact
policyengine_uk/data/fiscal_partition.yaml
- name: Measure national aggregates with the model fixed
if: ${{ !cancelled() }}
# A missing result fails the comparison, not the test job.
continue-on-error: true
run: |
tool="$RUNNER_TEMP/tool"
mkdir -p "$tool"
cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
model="policyengine-uk==$(uv run --frozen python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')"
PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact measure \
--label head --dataset-file policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "this pull request's build" \
--fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \
--no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true
cp "$RUNNER_TEMP/aggregate/result.json" aggregate_result.json \
|| echo "::warning::The aggregate measurement wrote no result; see the log above."
- name: Save the aggregate result (aggregates only)
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: aggregate_result.json
path: aggregate_result.json
# The policyengine-uk aggregate-impact tool (policyengine_uk/utils/
# aggregate_impact) at a pinned commit measures each build with the locked
# model, and the comparison job compares them with the model fixed
# (`compare --axis data`). Only result.json leaves a job; its private/
# folder (household detail) stays on the runner.
# The no-worsening check (W3). The base is built the same way as the head:
# TESTING, one clone and the head's lockfile, so both builds run one model
# and environment and the comparison measures the data change. As in the
# test job, only aggregate reports leave it: the fit summary, the
# distribution report and the aggregate result. No dataset or calibration
# log is uploaded, and nothing is cached: setup-uv restores uv's package
# cache and saves none.
base-build:
name: Build the base for comparison
if: false # DIAGNOSTIC ONLY (batch-3 attribution)
runs-on: ubuntu-latest
timeout-minutes: 60
env:
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
PYTHONUNBUFFERED: "1"
PE_UK_DATA_OA_CLONES: "1"
steps:
- name: Check out the head, for its lockfile
uses: actions/checkout@v6
with:
path: head
- name: Check out the base
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.base.sha }}
path: base
- name: Install uv
uses: astral-sh/setup-uv@v8.1.0
# Restore uv's package cache but never save one from a job that
# holds the data: its location can be moved (UV_CACHE_DIR).
with:
save-cache: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.13
- name: Sync the head's locked environment for the base
run: |
cp head/uv.lock base/uv.lock
cd base && uv sync --frozen --all-extras
- name: Download data inputs
working-directory: base
run: uv run --frozen make download
- name: Build the base datasets
working-directory: base
run: uv run --frozen make data
env:
TESTING: "1"
- name: Summarise the base's calibration fit
run: |
cp head/policyengine_uk_data/utils/fit_report.py base/policyengine_uk_data/utils/fit_report.py
uv run --frozen --project base python -m policyengine_uk_data.utils.fit_report summarise base/constituency_calibration_log.csv --out base_fit_summary.json
- name: Save the base fit summary
uses: actions/upload-artifact@v7
with:
name: base_fit_summary.json
path: base_fit_summary.json
- name: Measure the base's income distribution and single ages
working-directory: base
# The head's measurement code, on the base's build.
run: |
mkdir -p policyengine_uk_data/targets/reference_data
cp ../head/policyengine_uk_data/utils/distribution_validation.py policyengine_uk_data/utils/
cp ../head/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml policyengine_uk_data/targets/reference_data/
uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out ../base_distribution_report.json
- name: Save the base distribution report
uses: actions/upload-artifact@v7
with:
name: base_distribution_report.json
path: base_distribution_report.json
- name: Check out the aggregate-impact tool
if: ${{ !cancelled() }}
uses: actions/checkout@v6
with:
repository: PolicyEngine/policyengine-uk
ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
path: pe-uk-tool
sparse-checkout: |
policyengine_uk/utils/aggregate_impact
policyengine_uk/data/fiscal_partition.yaml
- name: Measure national aggregates with the model fixed
if: ${{ !cancelled() }}
run: |
tool="$RUNNER_TEMP/tool"
mkdir -p "$tool"
cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
model="policyengine-uk==$(uv run --frozen --project base python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')"
PYTHONPATH="$tool" uv run --frozen --project base python -m pe_uk_aggregate_impact measure \
--label base --dataset-file base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "the base's build" \
--fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \
--no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true
cp "$RUNNER_TEMP/aggregate/result.json" base_aggregate_result.json \
|| echo "::warning::The aggregate measurement wrote no result; see the log above."
- name: Save the aggregate result (aggregates only)
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: base_aggregate_result.json
path: base_aggregate_result.json
fit-comparison:
name: Calibration fit against the base
needs: [test, base-build]
# Report a failed side too: an earlier passing comment must not stand.
if: false # DIAGNOSTIC ONLY (batch-3 attribution)
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
issues: write
pull-requests: write
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v8.1.0
# Restore uv's package cache but never save one from a job that
# holds the data: its location can be moved (UV_CACHE_DIR).
with:
save-cache: false
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.13
- name: Sync locked environment
run: uv sync --frozen
- name: Download the fit summaries
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: "*fit_summary.json"
path: logs
merge-multiple: true
- name: Download the distribution reports
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: "*distribution_report.json"
path: reports
merge-multiple: true
- name: Download the aggregate results
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: "*aggregate_result.json"
path: aggregates
merge-multiple: true
- name: Check out the aggregate-impact tool
uses: actions/checkout@v6
with:
repository: PolicyEngine/policyengine-uk
ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
path: pe-uk-tool
sparse-checkout: |
policyengine_uk/utils/aggregate_impact
- name: Compare the final fits
id: compare
env:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
marker='<!-- policyengine-uk-data-fit-comparison -->'
if [ ! -f logs/fit_summary.json ] || [ ! -f logs/base_fit_summary.json ]; then
{
echo "$marker"
echo '## Calibration fit against the base build: FAIL'
echo
echo "One of the builds produced no fit summary, so the fit could not be compared. See the [workflow run]($RUN_URL)."
} > fit.md
echo "status=1" >> "$GITHUB_OUTPUT"
else
set +e
uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \
logs/base_fit_summary.json logs/fit_summary.json --markdown body.md
echo "status=$?" >> "$GITHUB_OUTPUT"
set -e
{ echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md
fi
if [ -f reports/base_distribution_report.json ] && [ -f reports/distribution_report.json ]; then
set +e
uv run --frozen python -m policyengine_uk_data.utils.distribution_validation compare \
reports/base_distribution_report.json reports/distribution_report.json --markdown distribution.md
distribution_status=$?
set -e
{ echo; cat distribution.md 2>/dev/null || echo '## Income distribution and single ages: FAIL (the comparison did not finish)'; } >> fit.md
if [ "$distribution_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
else
{ echo; echo '## Income distribution and single ages: FAIL'; echo; echo "The report is missing for one build, so the distributions could not be compared."; } >> fit.md
echo "status=1" >> "$GITHUB_OUTPUT"
fi
# National aggregates with the model fixed.
tool="$RUNNER_TEMP/tool"
mkdir -p "$tool" aggregates/base aggregates/head
cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
if [ -f aggregates/base_aggregate_result.json ] && [ -f aggregates/aggregate_result.json ]; then
mv aggregates/base_aggregate_result.json aggregates/base/result.json
mv aggregates/aggregate_result.json aggregates/head/result.json
set +e
PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact compare --axis data \
--base aggregates/base --head aggregates/head --markdown aggregates.md --run-url "$RUN_URL"
aggregate_status=$?
set -e
if [ "$aggregate_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
# The tool's comment marker belongs to policyengine-uk's own comment.
if [ -s aggregates.md ]; then
{ echo; sed '/^<!-- policyengine-uk-aggregate-impact -->$/d' aggregates.md; } >> fit.md
else
{ echo; echo '## National aggregates: FAIL (the comparison did not finish)'; } >> fit.md
echo "status=1" >> "$GITHUB_OUTPUT"
fi
else
{ echo; echo '## National aggregates: FAIL'; echo; echo "The result is missing for one build (see the build jobs' logs), so the aggregates could not be compared."; } >> fit.md
echo "status=1" >> "$GITHUB_OUTPUT"
fi
cat fit.md >> "$GITHUB_STEP_SUMMARY"
- name: Post or update the pull request comment
if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ github.event.pull_request.number }}
run: |
marker='<!-- policyengine-uk-data-fit-comparison -->'
if [ ! -s fit.md ]; then
# The job failed before the comparison could write a comment.
{
echo "$marker"
echo '## Calibration fit against the base build: FAIL'
echo
echo "The comparison did not run. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
} > fit.md
fi
existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
--jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \
| tail -n 1)
if [ -n "$existing" ]; then
gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@fit.md > /dev/null
else
gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@fit.md > /dev/null
fi
- name: Fail on a fit regression
if: steps.compare.outputs.status != '0'
run: |
echo "::error::The calibration fit, distribution or national aggregate comparison failed; see the job summary."
exit 1