diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml new file mode 100644 index 0000000000..7740c688a2 --- /dev/null +++ b/.github/workflows/aggregate-impact.yaml @@ -0,0 +1,225 @@ +# Measure a pull request's base and head on the certified dataset and post +# the comparison: receipts by tax head, spending and caseloads, poverty, +# inequality, deciles and the standard reform costings, 2025-26 to 2031-32. +# +# Both sides run the pull request head's copy of the measurement tool +# (policyengine_uk/utils/aggregate_impact), each in an environment synced +# from its own uv.lock, on the dataset pinned in +# policyengine_uk/tests/certified_dataset.yaml (bytes verified on each side). +# Each side reads its own fiscal partition. The job fails on a crash, a +# different dataset, non-finite figures or a check the head breaks that the +# base passes; moves above the flag thresholds are listed, not failed. +# +# DATA PROTECTION. The dataset is licensed survey microdata. Only aggregates +# may leave this job: the two result.json files, the comment and the summary. +# Never upload the measurement's private/ directory (household-level detail +# for the decile comparison), and never put the dataset in the Actions cache. +# Both sides are measured in one job so that detail stays on the runner. +# test_aggregate_impact_workflow.py enforces this. + +name: Aggregate impact + +on: + pull_request: + branches: + - main + paths: + - policyengine_uk/** + - pyproject.toml + - uv.lock + - .github/workflows/aggregate-impact.yaml + workflow_dispatch: + inputs: + base: + description: Base commit, branch or tag + default: main + head: + description: Head commit, branch or tag (default the dispatched ref) + default: "" + +concurrency: + group: aggregate-impact-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_PYTHON: "3.13" + PYTHONUNBUFFERED: "1" + +jobs: + report: + name: Aggregate impact + # Fork pull requests cannot read the private certified dataset. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Choose the commits + id: commits + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + INPUT_BASE: ${{ inputs.base }} + INPUT_HEAD: ${{ inputs.head }} + run: | + # A pull request's head is its merge commit, whose first parent is + # the base, so the comparison is exactly what merging would change. + if [ "$EVENT" = "pull_request" ]; then + base="$PR_BASE"; head="$GITHUB_SHA" + base_name="$PR_BASE_REF"; head_name="#$PR_NUMBER merged into $PR_BASE_REF" + else + base="${INPUT_BASE:-main}"; head="${INPUT_HEAD:-$GITHUB_SHA}" + base_name="$base"; head_name="${INPUT_HEAD:-$GITHUB_REF_NAME}" + fi + { + echo "base=$base" + echo "head=$head" + echo "base_name=$base_name" + echo "head_name=$head_name" + } >> "$GITHUB_OUTPUT" + + - name: Check out the head + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.head }} + path: head + + - name: Check out the base + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.base }} + path: base + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Sync each side's environment from its own lock + run: | + (cd base && uv sync --frozen --no-dev) + (cd head && uv sync --frozen --no-dev) + + - name: Measure base, then head + id: measure + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ACTOR: ${{ github.actor }} + BASE_NAME: ${{ steps.commits.outputs.base_name }} + HEAD_NAME: ${{ steps.commits.outputs.head_name }} + run: | + mkdir -p measured/base measured/head + if [ -z "$HUGGING_FACE_TOKEN" ]; then + if [ "$ACTOR" = "dependabot[bot]" ]; then + echo "::notice::Dependabot runs have no data access; the aggregate impact report is skipped." + echo '{"schema": "policyengine-uk-aggregate-impact/1", "status": "skipped", "reason": "Dependabot runs have no access to the certified dataset"}' \ + > measured/head/result.json + exit 0 + fi + echo "::error::HUGGING_FACE_TOKEN is not available, so the certified dataset cannot be read." + exit 1 + fi + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R head/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + measure() { + side="$1"; name="$2" + # A crash is recorded in result.json for the comparison to judge. + (cd "$side" && PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ + --label "$side" \ + --pin "$GITHUB_WORKSPACE/head/policyengine_uk/tests/certified_dataset.yaml" \ + --fallback-partition "$GITHUB_WORKSPACE/head/policyengine_uk/data/fiscal_partition.yaml" \ + --git-sha "$(git rev-parse HEAD)" \ + --git-ref "$name" \ + --out "$GITHUB_WORKSPACE/measured/$side") || true + } + measure base "$BASE_NAME" + measure head "$HEAD_NAME" + if [ ! -f measured/head/result.json ]; then + echo "::error::The head measurement wrote no result." + exit 1 + fi + + - name: Compare base with head + id: compare + run: | + mkdir -p report/base report/head + base_args=() + if [ -f measured/base/result.json ]; then + base_args=(--base measured/base) + cp measured/base/result.json report/base/result.json + fi + cp measured/head/result.json report/head/result.json + set +e + PYTHONPATH="$RUNNER_TEMP/tool" head/.venv/bin/python -m pe_uk_aggregate_impact compare \ + "${base_args[@]}" \ + --head measured/head \ + --markdown report/comment.md \ + --summary report/summary.json \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + if [ ! -s report/comment.md ]; then + # The comparison crashed: say so in place of any earlier result. + { + echo '' + echo '## Aggregate impact on certified data: FAIL' + echo + echo "The comparison did not finish, so this change has no report. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)." + } > report/comment.md + fi + cat report/comment.md >> "$GITHUB_STEP_SUMMARY" + # Household-level detail is no longer needed; remove it before any + # later step could pick it up. + rm -rf measured + + - name: Post or update the pull request comment + # Always, so a failed run replaces an earlier passing comment, even + # when it failed before the comparison could write one. + if: always() && github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + marker='' + if [ ! -s report/comment.md ]; then + mkdir -p report + { + echo "$marker" + echo '## Aggregate impact on certified data: FAIL' + echo + echo "The report did not run, so this change has no report. See the [workflow run]($RUN_URL)." + } > report/comment.md + fi + existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \ + | tail -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@report/comment.md > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@report/comment.md > /dev/null + fi + + - name: Upload the report (aggregates only) + if: always() + uses: actions/upload-artifact@v4 + with: + name: aggregate-impact-report + path: report + if-no-files-found: ignore + retention-days: 30 + + - name: Fail on a structural failure + if: steps.compare.outputs.status != '0' + run: | + echo "::error::The aggregate impact report found failures; see the job summary." + exit 1 diff --git a/changelog.d/2334-test-data.changed.md b/changelog.d/2334-test-data.changed.md new file mode 100644 index 0000000000..f84d1ecb8f --- /dev/null +++ b/changelog.d/2334-test-data.changed.md @@ -0,0 +1 @@ +- Microsimulation tests now run on the certified uk-data 1.58.1 dataset, pinned by commit and SHA-256, and fail instead of skipping when `POLICYENGINE_UK_REQUIRE_TEST_DATA` is set and the data is unavailable; the reform-cost fixtures are rebased onto it, keeping their earlier values as receipts. diff --git a/changelog.d/2334.added.md b/changelog.d/2334.added.md new file mode 100644 index 0000000000..c58137024c --- /dev/null +++ b/changelog.d/2334.added.md @@ -0,0 +1 @@ +- Added an aggregate-impact report (`python -m policyengine_uk.utils.aggregate_impact`) that measures a model checkout on the pinned certified dataset and compares base with head: receipts by tax head, spending and caseloads, poverty, inequality, deciles and the standard reform costings over 2025-26 to 2031-32, with accounting identities and a counted-once check against a reviewed fiscal partition (`policyengine_uk/data/fiscal_partition.yaml`). diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index a3a2dbad53..b506b44125 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -25,6 +25,22 @@ Use this skill whenever adding, moving, or reviewing tests. private H5 files in unit-style tests. Mock those seams or skip cleanly when the external artifact is unavailable. +## Microsimulation Data + +Microsimulation tests run on one certified dataset, pinned by Hugging Face +commit and SHA-256 in `policyengine_uk/tests/certified_dataset.yaml`. With +`HUGGING_FACE_TOKEN` set, `conftest.py` selects that dataset and verifies its +bytes before the first data test runs. Without a token, data tests skip. + +Set `POLICYENGINE_UK_REQUIRE_TEST_DATA=1` in jobs that must run the data tests. +Each data test then fails at setup when the dataset is unavailable, instead of +skipping, and runs only on the pinned dataset. Credential-free tests are +unaffected. + +The reform fixtures in `microsimulation/reforms_config.yaml` record the dataset +they were computed on. When the pin moves, recompute them on the new dataset, +and add each new value under `receipts` rather than overwriting the old ones. + ## Commands Run lint and formatting before committing: diff --git a/policyengine_uk/data/fiscal_partition.yaml b/policyengine_uk/data/fiscal_partition.yaml new file mode 100644 index 0000000000..cb5d415d0d --- /dev/null +++ b/policyengine_uk/data/fiscal_partition.yaml @@ -0,0 +1,964 @@ +# Fiscal partition: how the model's tax and spending totals divide into heads. +# +# Each head is one receipt (an OBR current-receipts line, split by who pays +# where the model splits it) and names the one variable that measures the +# household's share of it. Heads are mutually exclusive: no head's value +# contains another head's value, and no variable is summed twice in a total. +# +# The heads were specified from the OBR receipts classification and from each +# variable's formula, not copied from the model's own lists +# (GOV_TAX_VARIABLES, HOUSEHOLD_TAX_VARIABLES, +# PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES). +# policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py +# checks the two against each other without data, and the aggregate-impact +# report checks the accounting properties on the certified dataset. The #2159 +# fix adds tests of them on simulated households. +# +# Spending heads divide gov_spending and household_benefits the same way: +# one benefit or service per head, one variable each. +# +# This version describes main before the #2159 fix: expected_sdlt still adds +# corporate_sdlt, so the household SDLT head carries corporate SDLT too and +# corporate_sdlt is declared under model_double_counts. The fix gives +# corporate_sdlt its own head (sdlt_corporate) and empties that section. +# +# Consumers (tests, the per-PR aggregate impact report) read this file +# directly. Bump schema_version on any change to field meanings. +# +# Fields +# tax_heads[]: +# id one per head +# label sentence-case name +# variable the one model variable that measures the head +# entity household or person +# obr_line OBR current-receipts line the head belongs to, or +# not_an_obr_current_receipt +# report_group grouping for reports +# payer who bears it in the model: household (the household +# or its members pay it directly), employer, consumer +# incidence (allocated by consumption), corporate +# incidence (allocated by shareholding), or mixed +# totals the totals that include it +# sign nonnegative (at current-law parameter values, for +# non-negative inputs) or any (a change measure) +# zero_at_baseline expected: current-law parameters make it zero for +# every household (a reform-only head) +# reads other heads this head's formula reaches, with the +# route; reads that change eligibility, not inclusion +# exclusions[]: variables that must never be summed alongside the heads. +# kind: component (summed or read inside `head`), measure_within (an +# analytical decomposition already inside `head` through its inputs), +# composite (spans several heads), duplicate (same value as `head`), +# baseline_measure. +# model_double_counts[]: known violations still in the model: a variable a +# total lists although the head variable `contained_in` already adds it. +# spending_heads[]: as tax_heads, without obr_line and payer, plus +# caseload (default true): whether reports count recipients. +# overlap_guards[]: the places double counting could occur, and the +# property that should hold for each. The aggregate-impact report checks +# the geographic_exclusivity guards on the certified dataset; the #2159 +# fix's tests check the others. +schema_version: 1 + +totals: + gov_tax: + label: Government tax revenue in respect of the household + description: Receipts at their level, including employer-paid taxes and VAT. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + household_tax: + label: Taxes borne by the household + description: >- + Taxes subtracted from household net income. VAT and employer NI enter + only as reform-induced changes (vat_change and the employer NI response + incidence), because baseline VAT and employer NI are not deducted from + the incomes the model starts from. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + pre_budget_change_household_tax: + label: Taxes borne by the household, pre-budget-change definition + description: >- + A fixed earlier definition of household_tax used by + pre_budget_change_household_net_income. It omits heads added since + (alcohol, tobacco and vehicle excise duties, private school VAT, the + high value council tax surcharge, student loan repayments, and the + incidence-change heads). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + + gov_spending: + label: Government spending in respect of the household + description: >- + Cash benefits, childcare and student support, and public services in + kind (NHS, education, transport subsidies). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + household_benefits: + label: Benefits received by the household + description: >- + The benefit side of household net income. It is gov_spending without the + contributed budget-change line, which is not paid to households. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + +tax_heads: + # Income and earnings + - id: income_tax + label: Income tax + variable: income_tax + entity: person + obr_line: Income tax + report_group: income_tax + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_individual + label: National Insurance (employee and self-employed) + variable: national_insurance + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_employer + label: National Insurance (employer) + variable: ni_employer + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: employer + totals: [gov_tax] + sign: nonnegative + - id: employer_nics_response_capital + label: Employer NI change borne by capital + variable: employer_ni_response_capital_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: corporate incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: employer_nics_response_consumer + label: Employer NI change borne by consumers + variable: employer_ni_response_consumer_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: consumer incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: capital_gains_tax + label: Capital gains tax + variable: capital_gains_tax + entity: person + obr_line: Capital gains tax + report_group: capital_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: student_loan_repayments + label: Student loan repayments + variable: student_loan_repayments + entity: person + obr_line: not_an_obr_current_receipt + obr_line_note: >- + A financial transaction: repayments reduce the student loan book, not + borrowing, so they are outside current receipts. The model counts them + in gov_tax and household_tax; reports should show them separately. + report_group: financial_transactions + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + + # Consumption + - id: vat + label: VAT + variable: vat + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [gov_tax] + sign: nonnegative + - id: vat_change + label: Change in VAT from its baseline + variable: vat_change + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [household_tax, pre_budget_change_household_tax] + sign: any + zero_at_baseline: expected + reads: + - head: vat + route: vat_change adds vat and subtracts baseline_vat + why: a change measure built from the level; the two are never in the same total + - id: private_school_vat + label: VAT on private school fees + variable: private_school_vat + entity: household + obr_line: Value added tax + report_group: vat + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + zero_at_baseline: expected + zero_at_baseline_note: >- + gov.contrib.labour.private_school_vat is 0 at current law in this + model, so the head is reform-only here. + reads: + - head: income_tax + route: private_school_vat -> attends_private_school -> income_tax + why: attendance is imputed from income; it decides who attends, not how much tax is counted + - head: nics_individual + route: private_school_vat -> attends_private_school -> (net income) -> national_insurance + why: as above + - head: council_tax + route: private_school_vat -> attends_private_school -> household_benefits -> council_tax_benefit -> council_tax + why: as above + - id: fuel_duty + label: Fuel duties + variable: fuel_duty + entity: household + obr_line: Fuel duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: alcohol_duty + label: Alcohol duties + variable: alcohol_duty + entity: household + obr_line: Alcohol duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tobacco_duty + label: Tobacco duties + variable: tobacco_duty + entity: household + obr_line: Tobacco duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: vehicle_excise_duty + label: Vehicle excise duties + variable: vehicle_excise_duty + entity: household + obr_line: Vehicle excise duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tv_licence + label: TV licence fee + variable: tv_licence + entity: household + obr_line: Licence fee receipts + report_group: other_receipts + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + reads: + - head: income_tax + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> income_tax + why: the over-75 discount depends on Pension Credit receipt, a means test on net income + - head: nics_individual + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> national_insurance + why: as above + + # Property + - id: council_tax + label: Council tax + variable: council_tax + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: high_value_council_tax_surcharge + label: High value council tax surcharge + variable: high_value_council_tax_surcharge + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + - id: domestic_rates + label: Northern Ireland domestic rates + variable: domestic_rates + entity: household + obr_line: Council tax + obr_line_note: >- + Northern Ireland's domestic rates are its counterpart to council tax; + check the line against the OBR table in use before publishing a + line-by-line comparison. + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: business_rates + label: Business rates + variable: business_rates + entity: household + obr_line: Business rates + report_group: business_rates + payer: corporate incidence + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: sdlt_household + label: Stamp duty land tax (own purchases, and corporate until the #2159 fix) + variable: expected_sdlt + entity: household + obr_line: Stamp duty land tax + report_group: property_transaction_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + Before the #2159 fix expected_sdlt adds corporate_sdlt and a stray 0.045 + per household, so this head is both parts of the receipt. + - id: land_transaction_tax + label: Land transaction tax (Wales) + variable: expected_ltt + entity: household + obr_line: Land transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_ltt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + - id: land_and_buildings_transaction_tax + label: Land and buildings transaction tax (Scotland) + variable: expected_lbtt + entity: household + obr_line: Land and buildings transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_lbtt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + + # Reform-only heads + - id: corporate_incident_tax_change + label: Unspecified tax change borne by capital + variable: corporate_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by shareholding. + report_group: incidence_changes + payer: corporate incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: consumer_incident_tax_change + label: Unspecified tax change borne by consumers + variable: consumer_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by consumption. + report_group: incidence_changes + payer: consumer incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: wealth_tax + label: Wealth tax + variable: wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: non_primary_residence_wealth_tax + label: Wealth tax excluding the main residence + variable: non_primary_residence_wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: land_value_tax + label: Land value tax + variable: LVT + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: carbon_tax + label: Carbon tax + variable: carbon_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + +exclusions: + # Property transaction taxes + - {variable: stamp_duty_land_tax, kind: component, head: sdlt_household, note: "Summed inside expected_sdlt, which until the #2159 fix also adds corporate_sdlt and a stray 0.045; not zeroed by gov.hmrc.stamp_duty.abolish. Report the head, not this."} + - {variable: sdlt_on_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_rent, kind: component, head: sdlt_household} + - {variable: land_transaction_tax, kind: component, head: land_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: ltt_on_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: land_and_buildings_transaction_tax, kind: component, head: land_and_buildings_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: lbtt_on_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: corporate_tax_incidence, kind: composite, spans: [sdlt_household, business_rates]} + - {variable: hbai_excluded_income, kind: composite, spans: [sdlt_household, business_rates], note: "Minus corporate_tax_incidence."} + - {variable: baseline_business_rates, kind: duplicate, head: business_rates, note: "Same formula and current-law parameters as business_rates."} + # Income tax and National Insurance + - {variable: income_tax_pre_charges, kind: component, head: income_tax} + - {variable: earned_income_tax, kind: component, head: income_tax} + - {variable: savings_income_tax, kind: component, head: income_tax} + - {variable: dividend_income_tax, kind: component, head: income_tax} + - {variable: property_income_tax, kind: component, head: income_tax} + - {variable: CB_HITC, kind: component, head: income_tax} + - {variable: personal_pension_contributions_tax, kind: component, head: income_tax} + - {variable: ni_employee, kind: measure_within, head: nics_individual, note: "Class 1 employee; national_insurance adds the classes directly."} + - {variable: ni_self_employed, kind: measure_within, head: nics_individual, note: "Classes 2 and 4; national_insurance adds the classes directly."} + - {variable: ni_class_1_employee, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_primary, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_additional, kind: component, head: nics_individual} + - {variable: ni_class_2, kind: component, head: nics_individual} + - {variable: ni_class_3, kind: component, head: nics_individual} + - {variable: ni_class_4, kind: component, head: nics_individual} + - {variable: ni_class_1_employer, kind: component, head: nics_employer} + - {variable: salary_sacrifice_pension_ni_employee, kind: measure_within, head: nics_individual, note: "The excess sacrifice is added back to employment_income, so Class 1 already charges it."} + - {variable: salary_sacrifice_pension_ni_employer, kind: measure_within, head: nics_employer, note: "As above, for employer Class 1."} + - {variable: tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: benunit_tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: total_national_insurance, kind: composite, spans: [nics_individual, nics_employer]} + - {variable: student_loan_repayment, kind: component, head: student_loan_repayments} + # Consumption + - {variable: baseline_vat, kind: baseline_measure, head: vat_change, note: "Subtracted inside vat_change."} + - {variable: beer_duty, kind: component, head: alcohol_duty} + - {variable: cider_duty, kind: component, head: alcohol_duty} + - {variable: sparkling_cider_duty, kind: component, head: alcohol_duty} + - {variable: wine_duty, kind: component, head: alcohol_duty} + - {variable: spirits_duty, kind: component, head: alcohol_duty} + - {variable: other_fermented_duty, kind: component, head: alcohol_duty} + - {variable: car_vehicle_excise_duty, kind: component, head: vehicle_excise_duty} + # Council tax net of a benefit + - {variable: council_tax_less_benefit, kind: composite, spans: [council_tax], note: "Council tax minus Council Tax Reduction, a benefit; not a receipt."} + +model_double_counts: + - variable: corporate_sdlt + contained_in: expected_sdlt + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + issue: https://github.com/PolicyEngine/policyengine-uk/issues/2159 + +overlap_guards: + - id: sdlt_payer_split + kind: payer_split + heads: [sdlt_household] + property: >- + Until the #2159 fix, expected_sdlt adds corporate_sdlt, which the totals + also list (model_double_counts). The fix splits the head in two, neither + reading the other, both zeroed by gov.hmrc.stamp_duty.abolish. + - id: property_transaction_tax_geography + kind: geographic_exclusivity + heads: [sdlt_household, land_transaction_tax, land_and_buildings_transaction_tax] + property: >- + At most one is non-zero for any household (England and Northern Ireland, + Wales, Scotland). Until the #2159 fix this does not hold: expected_sdlt + carries a stray 0.045, and corporate SDLT for shareholders, in every + nation. + - id: local_tax_geography + kind: geographic_exclusivity + heads: [council_tax, domestic_rates] + property: >- + No household has both. Both are inputs, so this is a property of the + data, checked on the certified dataset. + - id: vat_level_and_change + kind: level_and_change + heads: [vat, vat_change] + property: Never in the same total. vat_change reads vat. + - id: employer_nics_level_and_incidence + kind: level_and_change + heads: [nics_employer, employer_nics_response_capital, employer_nics_response_consumer] + property: The level is only in gov_tax; the response incidence is only in household_tax. + - id: corporate_incidence_allocations + kind: shared_allocation_key + heads: [business_rates, sdlt_household, corporate_incident_tax_change, carbon_tax, land_value_tax] + property: >- + Each allocates a different receipt by shareholding; none reads another. + corporate_tax_incidence sums two of them and is excluded. + - id: private_school_vat_and_vat + kind: possible_base_overlap + heads: [vat, private_school_vat] + property: >- + vat applies to imputed consumption; if that consumption includes school + fees, turning on private_school_vat would tax them twice. The head is + zero at baseline, so baseline totals are unaffected; check the + consumption imputation before costing a reform that sets it. + - id: eligibility_reads + kind: read_not_inclusion + heads: [tv_licence, private_school_vat] + property: >- + These heads reach other heads of the same total only through + eligibility or imputation (see each head's reads). Raising the other + head by a small amount leaves them unchanged on the test population. + +spending_heads: + # Pensioners + - id: state_pension + label: State Pension + variable: state_pension + entity: person + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pension_credit + label: Pension Credit + variable: pension_credit + entity: benunit + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: winter_fuel_payment + label: Winter Fuel Payment + variable: winter_fuel_allowance + entity: household + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pawhp + label: Pension Age Winter Heating Payment (Scotland) + variable: pawhp + entity: household + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Working-age means-tested benefits + - id: universal_credit + label: Universal Credit + variable: universal_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: housing_benefit + label: Housing Benefit + variable: housing_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: council_tax_reduction + label: Council Tax Reduction + variable: council_tax_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_income + label: Employment and Support Allowance (income-related) + variable: esa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_contrib + label: Employment and Support Allowance (contribution-based) + variable: esa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_income + label: Jobseeker's Allowance (income-based) + variable: jsa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_contrib + label: Jobseeker's Allowance (contribution-based) + variable: jsa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: income_support + label: Income Support + variable: income_support + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: working_tax_credit + label: Working Tax Credit + variable: working_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: child_tax_credit + label: Child Tax Credit + variable: child_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: incapacity_benefit + label: Incapacity Benefit + variable: incapacity_benefit + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Disability and carers + - id: pip + label: Personal Independence Payment + variable: pip + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: dla + label: Disability Living Allowance + variable: dla + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: attendance_allowance + label: Attendance Allowance + variable: attendance_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carers_allowance + label: Carer's Allowance + variable: carers_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carer_support_payment + label: Carer Support Payment (Scotland) + variable: carer_support_payment + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_carer_supplement + label: Carer's Allowance Supplement (Scotland) + variable: scottish_carer_supplement + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: sda + label: Severe Disablement Allowance + variable: sda + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: iidb + label: Industrial Injuries Disablement Benefit + variable: iidb + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: afcs + label: Armed Forces Compensation Scheme + variable: afcs + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: war_widows_pension + label: War Widow's or Widower's Pension + variable: war_widows_pension + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Children and families + - id: child_benefit + label: Child Benefit + variable: child_benefit + entity: benunit + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_child_payment + label: Scottish Child Payment + variable: scottish_child_payment + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: two_child_limit_payment + label: Two Child Limit Payment (Scotland, counterfactual) + variable: two_child_limit_payment + entity: benunit + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: healthy_start_vouchers + label: Healthy Start vouchers + variable: healthy_start_vouchers + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: maternity_allowance + label: Maternity Allowance + variable: maternity_allowance + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_maternity_pay + label: Statutory Maternity Pay + variable: statutory_maternity_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_paternity_pay + label: Statutory Paternity Pay + variable: statutory_paternity_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_sick_pay + label: Statutory Sick Pay + variable: statutory_sick_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: ssmg + label: Sure Start Maternity Grant + variable: ssmg + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bsp + label: Bereavement Support Payment + variable: bsp + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Childcare and education support + - id: tax_free_childcare + label: Tax-Free Childcare + variable: tax_free_childcare + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: extended_childcare_entitlement + label: Extended childcare entitlement + variable: extended_childcare_entitlement + entity: benunit + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: universal_childcare_entitlement + label: Universal childcare entitlement + variable: universal_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: targeted_childcare_entitlement + label: Targeted childcare entitlement + variable: targeted_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: care_to_learn + label: Care to Learn + variable: care_to_learn + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: childcare_grant + label: Childcare Grant + variable: childcare_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: parents_learning_allowance + label: Parents' Learning Allowance + variable: parents_learning_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: adult_dependants_grant + label: Adult Dependants' Grant + variable: adult_dependants_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: travel_grant + label: Travel Grant + variable: travel_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: disabled_students_allowance + label: Disabled Students' Allowance + variable: disabled_students_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bursary_fund_16_to_19 + label: 16 to 19 Bursary Fund + variable: bursary_fund_16_to_19 + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + # Public services in kind + - id: nhs_spending + label: NHS spending + variable: nhs_spending + entity: person + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dfe_education_spending + label: State education spending + variable: dfe_education_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dft_subsidy_spending + label: Transport subsidy spending + variable: dft_subsidy_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + # Cost-of-living and energy support (none in force at current law) + - id: cost_of_living_support_payment + label: Cost-of-living support payment + variable: cost_of_living_support_payment + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: energy_bills_rebate + label: Energy Bills Rebate + variable: energy_bills_rebate + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: epg_subsidy + label: Energy Price Guarantee subsidy + variable: epg_subsidy + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + # Reform-only spending + - id: basic_income + label: Basic income + variable: basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: disability_basic_income + label: Disability basic income + variable: disability_basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: other_public_spending_budget_change + label: Other public spending budget change + variable: other_public_spending_budget_change + entity: household + report_group: hypothetical_spending + totals: [gov_spending] + sign: any + zero_at_baseline: expected + caseload: false diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py new file mode 100644 index 0000000000..0831852567 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -0,0 +1,214 @@ +"""The aggregate-impact measurement on the certified dataset. + +One year, the counted-once check on every tax component plus a few benefits. +Runs where microsimulation tests run (see conftest's data mode). +""" + +import dataclasses + +import numpy as np +import pytest + +from policyengine_uk.utils.aggregate_impact.measure import measure +from policyengine_uk.utils.aggregate_impact.partition import ( + default_partition_path, + load_partition, +) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parameter_node +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +pytestmark = pytest.mark.microsimulation + +YEAR = 2026 +# Components whose rise raises another benefit: a known model defect each. +# When one is fixed this test fails until its entry is removed. Empty since +# #2365 counted statutory sick, maternity and paternity pay as UC earnings. +KNOWN_RISING_WITH_INCOME: set = set() +# incapacity_benefit raises Universal Credit only through Carer's Allowance +# (#2424), which the check must report as mediated, not fail. +SPENDING_SAMPLE = { + "state_pension", + "universal_credit", + "child_benefit", + "statutory_sick_pay", + "incapacity_benefit", +} + + +@pytest.fixture(scope="module") +def measured(): + from policyengine_uk import Microsimulation + + sim = Microsimulation() + dataset = sim.dataset + partition = load_partition(default_partition_path()) + spec = dataclasses.replace( + load_spec(), first_year=YEAR, last_year=YEAR, guard_year=YEAR + ) + tax_components = {h.variable for h in partition.tax_heads} | { + d.variable for d in partition.double_counts + } + result, households = measure( + sim, + lambda reform=None: Microsimulation(dataset=dataset, reform=reform), + partition, + spec, + label="test", + reforms=[r for r in load_reforms() if r.scale], + guard_variables=tax_components | SPENDING_SAMPLE, + ) + return result, households, partition, Microsimulation(dataset=dataset) + + +def checks(result, prefix): + return [c for c in result["checks"] if c["id"].startswith(prefix)] + + +def test_accounting_identities_and_closure_hold(measured): + result, _, _, _ = measured + assert result["status"] == "ok" + for prefix in ( + "closure", + "identity:", + "finite:", + "weights:", + "bounds:", + "determinism:", + ): + failing = [c for c in checks(result, prefix) if c["status"] != "pass"] + assert failing == [], failing + + +def test_every_tax_component_is_counted_exactly_once(measured): + result, _, partition, _ = measured + guard = result["guard"] + assert guard["status"] == "ran" and guard["reset_reproduces_baseline"] + rows = {r["variable"]: r for r in guard["components"]} + for head in partition.tax_heads: + row = rows[head.variable] + assert row["status"] == "pass", row + for total, k in row["coefficients"].items(): + assert k == pytest.approx(1, abs=1e-3), (head.id, total) + + +def test_declared_double_counts_are_still_double_counted_by_the_model(measured): + result, _, partition, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for declared in partition.double_counts: + present = next( + d + for d in result["guard"]["double_counts"] + if d["variable"] == declared.variable + ) + assert present["status"] == "pass", present + for total in declared.totals: + assert rows[declared.variable]["model_coefficients"][ + total + ] == pytest.approx(2, abs=1e-3) + + +def test_spending_never_rises_by_more_than_is_added_except_known_defects(measured): + result, _, _, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for variable in SPENDING_SAMPLE: + row = rows[variable] + if variable in KNOWN_RISING_WITH_INCOME: + assert ( + row["status"] == "fail" + and 1 < row["coefficients"]["gov_spending"] < 1.05 + ) + else: + assert row["status"] == "pass", row + + +def test_head_totals_agree_with_the_model_weighted_sums(measured): + result, _, partition, sim = measured + metrics = {m["id"]: m for m in result["metrics"]} + for head in partition.heads: + section = "receipts" if head.kind == "tax" else "spending" + metric = metrics[f"{section}.{head.id}"] + if str(YEAR) in metric.get("suppressed", ()): + # Too few records to publish (privacy.py): only the count is checked. + records = int(np.count_nonzero(sim.calculate(head.variable, YEAR).values)) + assert 0 < records < 10, head.id + continue + ours = metric["values"][str(YEAR)] + theirs = float(sim.calculate(head.variable, YEAR).sum()) + assert ours == pytest.approx(theirs, rel=1e-6, abs=1e3), head.id + receipts = metrics["balance.receipts"]["values"][str(YEAR)] + model = metrics["model.gov_tax"]["values"][str(YEAR)] + double_counted = sum( + metrics[f"double_counts.{d.variable}"]["values"][str(YEAR)] + for d in partition.double_counts + if "gov_tax" in d.totals + ) + assert receipts + double_counted == pytest.approx(model, rel=1e-9) + + +def test_household_detail_is_complete(measured): + _, households, _, _ = measured + columns = dict(households[str(YEAR)]) + # The suppressed components the income leaves out: metadata, not a + # per-household column. + excludes = columns.pop("excludes") + assert excludes.ndim == 1 and excludes.dtype.kind == "U" + n = len(columns["household_id"]) + assert all(len(v) == n for v in columns.values()) + # Scalars only: a failing assertion on an array would print household + # values into public CI logs. + non_finite = int((~np.isfinite(columns["income"])).sum()) + assert non_finite == 0, f"{non_finite} households have a non-finite income" + outside = int((~np.isin(columns["decile"], np.arange(-1, 11))).sum()) + assert outside == 0, f"{outside} households have a decile outside -1 to 10" + + +def test_relative_reforms_scale_their_parameters_and_move_their_targets(measured): + result, _, _, sim = measured + passing = {c["id"] for c in result["checks"] if c["status"] == "pass"} + relative = [r for r in load_reforms() if r.scale] + assert {r.id for r in relative} == { + "uc_standard_allowance_plus_10pc", + "state_pension_plus_10pc", + } + outcomes = {o["id"]: o for o in result["reforms"]} + for reform in relative: + assert f"reform_runs:{reform.id}" in passing + assert f"responsive:{reform.id}" in passing + for path, factor in reform.scale.items(): + baseline = parameter_node(sim.tax_benefit_system.parameters, path)( + f"{YEAR}-01-01" + ) + assert outcomes[reform.id]["parameters"][path][str(YEAR)] == pytest.approx( + baseline * factor + ) + + +def test_the_published_result_holds_aggregates_only(measured): + """result.json may be public: no household-length arrays, no single values.""" + result, households, _, _ = measured + n_households = len(households[str(YEAR)]["household_id"]) + + def numeric_lists(node): + if isinstance(node, dict): + for value in node.values(): + yield from numeric_lists(value) + elif isinstance(node, list): + if node and all(isinstance(v, (int, float)) for v in node): + yield node + for value in node: + yield from numeric_lists(value) + + longest = max((len(values) for values in numeric_lists(result)), default=0) + assert longest <= 50 < n_households + details = " ".join(d for c in result["checks"] for d in c["details"]) + assert "minimum £" not in details and "largest gap" not in details + + +def test_geographic_guards_hold_once_declared_double_counts_are_netted(measured): + """Council tax and domestic rates never meet; nor do SDLT, LTT and LBTT + once corporate SDLT, which expected_sdlt carries while #2159 is open, is + netted out of it as a declared double count.""" + result, _, _, _ = measured + status = {c["id"]: c["status"] for c in result["checks"]} + assert status["overlap_guard:local_tax_geography"] == "pass" + assert status["overlap_guard:property_transaction_tax_geography"] == "pass" diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py new file mode 100644 index 0000000000..5aeb771469 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py @@ -0,0 +1,886 @@ +"""Comparison and rendering of two measurements, on synthetic results.""" + +import copy +import json +import math + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact import COMMENT_MARKER, TOOL_SCHEMA +from policyengine_uk.utils.aggregate_impact.compare import ( + REQUIRED_CHECKS, + _match, + classify_check, + compare, + decile_impacts, +) +from policyengine_uk.tests.aggregate_impact.test_aggregate_impact_measure import ( + _rebuilt, +) +from policyengine_uk.utils.aggregate_impact.measure import weighted_gini +from policyengine_uk.utils.aggregate_impact.render import fmt_change, fmt_level, render +from policyengine_uk.utils.aggregate_impact.spec import Threshold, load_spec + +SPEC = load_spec() +YEARS = SPEC.years +SHA = "5" * 64 + +KINDS = { + "receipts": "gbp", + "caseloads": "count", + "poverty": "rate", + "inequality": "index", +} +finite = st.floats(-1e12, 1e12, allow_nan=False, allow_infinity=False) + + +def check(check_id, status="pass", failing=()): + return { + "id": check_id, + "label": check_id, + "section": "s", + "severity": "fail", + "status": status, + "details": ["d"] if status != "pass" else [], + "failing": { + scope: "d" for scope in (failing or ([""] if status == "fail" else [])) + }, + } + + +def result(values, checks=(), sha=SHA, status="ok", partition_sha="p", required=True): + """A measurement with one metric per (section, name) and the given checks.""" + all_checks = [check(c) for c in REQUIRED_CHECKS] if required else [] + for entry in checks: + all_checks.append(check(*entry) if isinstance(entry, tuple) else entry) + return { + "schema": TOOL_SCHEMA, + "status": status, + "label": "x", + "years": YEARS, + "provenance": { + "dataset": {"verified_sha256": sha, "path": "d.h5"}, + "git_sha": "abc", + }, + "runtime": { + "distributions": ["numpy==2.0"], + "python": "3.13", + "machine": "arm64", + }, + "partition": {"sha256": partition_sha, "source": "s"}, + "metrics": [ + { + "id": f"{section}.{name}", + "label": name, + "section": section, + "group": None, + "kind": KINDS[section], + "unit": None, + "values": {str(y): v for y, v in zip(YEARS, series)}, + } + for (section, name), series in values.items() + ], + "checks": all_checks, + "guard": {"status": "skipped", "reason": "test"}, + "reforms": [], + "timings": {"total": 1}, + } + + +metric_values = st.dictionaries( + st.tuples(st.sampled_from(sorted(KINDS)), st.sampled_from(["a", "b", "c"])), + st.lists(finite, min_size=len(YEARS), max_size=len(YEARS)), + min_size=1, + max_size=6, +) + + +@pytest.mark.parametrize( + "base, head, blocking, expected", + [ + ("pass", "pass", False, "pass"), + ("fail", "pass", False, "fixed"), + ("warn", "pass", False, "fixed"), + (None, "pass", False, "pass"), + ("pass", "fail", False, "regression"), + (None, "fail", False, "regression"), + ("skipped", "fail", False, "regression"), + ("fail", "fail", False, "pre_existing"), + ("fail", "fail", True, "regression"), + ("warn", "fail", False, "regression"), + ("pass", "warn", False, "new_warning"), + ("warn", "warn", False, "warning"), + ("pass", "skipped", False, "skipped"), + ("pass", None, False, "removed"), + ], +) +def test_check_classification(base, head, blocking, expected): + assert classify_check(base, head, blocking) == expected + + +@settings(max_examples=200, deadline=None) +@given( + base=st.sets(st.sampled_from(["2025", "2026", "2027", ""])), + head=st.sets(st.sampled_from(["2025", "2026", "2027", ""]), min_size=1), +) +def test_a_failure_is_pre_existing_only_if_the_base_fails_everywhere_the_head_does( + base, head +): + verdict = classify_check("fail" if base else "pass", "fail", False, base, head) + assert verdict == ("pre_existing" if base and head <= base else "regression") + + +@settings(max_examples=60, deadline=None) +@given(values=metric_values) +def test_identical_sides_change_nothing(values): + side = result(values, checks=[("identity:x", "pass"), ("sign:y", "fail")]) + summary = compare(side, copy.deepcopy(side), SPEC) + assert summary["status"] == "pass", summary["failures"] + assert summary["flags"] == [] + assert all(not row["changed"] for row in summary["rows"]) + assert {c["classification"] for c in summary["checks"]} == {"pass", "pre_existing"} + + +@settings(max_examples=60, deadline=None) +@given(base=metric_values, data=st.data()) +def test_changes_are_antisymmetric(base, data): + head = {k: [v + data.draw(finite) for v in series] for k, series in base.items()} + forward = {r["id"]: r for r in compare(result(base), result(head), SPEC)["rows"]} + backward = {r["id"]: r for r in compare(result(head), result(base), SPEC)["rows"]} + for metric_id, row in forward.items(): + for year, cell in row["values"].items(): + assert cell["delta"] == pytest.approx( + -backward[metric_id]["values"][year]["delta"], abs=1e-3 + ) + + +@settings(max_examples=200, deadline=None) +@given(base=finite, head=finite, absolute=st.floats(0, 1e9), relative=st.floats(0, 0.5)) +def test_a_move_is_flagged_exactly_when_it_exceeds_the_larger_bound( + base, head, absolute, relative +): + threshold = Threshold(absolute, relative) + assert threshold.exceeded(base, head) == ( + abs(head - base) > max(absolute, relative * abs(base)) + ) + + +def test_flags_follow_the_spec_thresholds(): + base = {("receipts", "a"): [100e9] * len(YEARS)} + small = {("receipts", "a"): [100e9 + 50e6] * len(YEARS)} + large = {("receipts", "a"): [100e9 + 200e6] * len(YEARS)} + assert compare(result(base), result(small), SPEC)["flags"] == [] + assert len(compare(result(base), result(large), SPEC)["flags"]) == len(YEARS) + + +CRASHED = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "error": "Traceback\nValueError: boom", +} +ONE = {("receipts", "a"): [1.0] * len(YEARS)} + + +def test_a_crashed_or_missing_head_fails(): + summary = compare(result(ONE), CRASHED, SPEC) + assert summary["status"] == "fail" and "boom" in summary["failures"][0] + assert compare(result(ONE), None, SPEC)["status"] == "fail" + + +def test_a_crashed_or_missing_base_fails_unless_it_is_not_required(): + for base in (CRASHED, None): + assert compare(base, result(ONE), SPEC)["status"] == "fail" + relaxed = compare(base, result(ONE), SPEC, require_base=False) + assert relaxed["status"] == "pass" and relaxed["warnings"] + + +def test_without_a_base_the_heads_own_failing_checks_block(): + head = result(ONE, checks=[("identity:x", "fail")]) + summary = compare(None, head, SPEC, require_base=False) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_different_datasets_fail(): + summary = compare(result(ONE), result(ONE, sha="6" * 64), SPEC) + assert summary["status"] == "fail" + + +def test_non_finite_head_figures_block_even_when_the_base_shares_them(): + checks = [check("finite:x", "fail")] + summary = compare(result(ONE, checks), result(ONE, checks), SPEC) + assert summary["status"] == "fail" + + +def test_a_non_finite_head_figure_fails_even_if_no_check_caught_it(): + """Reform costings are measured after most checks; compare looks itself.""" + bad = {("receipts", "a"): [1.0] * (len(YEARS) - 1) + [float("nan")]} + summary = compare(result(ONE), result(bad), SPEC) + assert summary["status"] == "fail" + assert any("not finite" in f for f in summary["failures"]) + + +def test_a_newly_failing_year_is_a_regression_not_pre_existing(): + base = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + same = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + worse = result(ONE, checks=[check("identity:x", "fail", ["2025", "2026"])]) + assert compare(base, same, SPEC)["status"] == "pass" + summary = compare(base, worse, SPEC) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_a_check_that_vanishes_fails_unless_the_partition_changed(): + base = result(ONE, checks=[("sign:vat", "pass")]) + summary = compare(base, result(ONE), SPEC) + assert summary["status"] == "fail" + assert any("sign:vat" in f for f in summary["failures"]) + edited = compare(base, result(ONE, partition_sha="q"), SPEC) + assert edited["status"] == "pass" + assert any("sign:vat" in w for w in edited["warnings"]) + + +def test_a_head_without_the_required_checks_fails(): + summary = compare(result(ONE), result(ONE, required=False), SPEC) + assert summary["status"] == "fail" + assert any("finite:metrics" in f for f in summary["failures"]) + + +def test_a_check_that_no_longer_applies_is_noted_not_failed(): + base = result(ONE, checks=[("counted_once:x", "pass")]) + head = result(ONE, checks=[check("counted_once:x", "skipped")]) + summary = compare(base, head, SPEC) + assert summary["status"] == "pass" + assert any("counted_once:x" in w for w in summary["warnings"]) + + +def test_a_figure_on_one_side_only_is_a_change(): + two = {**ONE, ("receipts", "b"): [2.0] * len(YEARS)} + removed = compare(result(two), result(ONE), SPEC) + row = next(r for r in removed["rows"] if r["id"] == "receipts.b") + assert row["changed"] and row["only_in"] == "base" + text = render(removed, SPEC) + assert "removed: base levels" in text + assert "No change in any year:** receipts" not in text + added = compare(result(ONE), result(two), SPEC) + assert "new: head levels" in render(added, SPEC) + + +def test_strict_mode_fails_pre_existing_failures(): + side = result(ONE, checks=[("sign:vat", "fail")]) + assert compare(side, copy.deepcopy(side), SPEC)["status"] == "pass" + assert compare(side, copy.deepcopy(side), SPEC, strict=True)["status"] == "fail" + + +def test_a_partition_edit_is_noted(): + summary = compare(result(ONE), result(ONE, partition_sha="q"), SPEC) + assert any("partition" in w for w in summary["warnings"]) + + +def test_households_match_only_one_to_one(): + assert _match(np.array([1, 2]), np.array([1, 2, 3])) is None + assert _match(np.array([1, 2]), np.array([1, 1])) is None + assert _match(np.array([1, 1]), np.array([1, 1])) is None + assert np.array_equal(_match(np.array([1, 2, 3]), np.array([3, 1, 2])), [1, 2, 0]) + + +@settings(max_examples=100, deadline=None) +@given( + ids=st.lists(st.integers(0, 10_000), min_size=1, max_size=60, unique=True), + data=st.data(), +) +def test_matching_recovers_any_permutation(ids, data): + base = np.array(ids) + head = np.array(data.draw(st.permutations(ids))) + matched = _match(base, head) + assert matched is not None and np.array_equal(head[matched], base) + + +def households(n, rng, decile=None, income=None, weight=None): + return { + str(y): { + "household_id": np.arange(n), + "household_weight": weight + if weight is not None + else rng.uniform(1, 1000, n), + "household_count_people": rng.integers(1, 6, n).astype(float), + "decile": decile if decile is not None else rng.integers(1, 11, n), + "income": income if income is not None else rng.uniform(1e3, 1e5, n), + } + for y in YEARS + } + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(20, 400)) +def test_decile_impacts_match_a_direct_grouped_calculation(seed, n): + rng = np.random.default_rng(seed) + base = households(n, rng) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + rng.normal(0, 500, n) + impacts, _ = decile_impacts(base, head, YEARS) + year = str(YEARS[0]) + b, h = base[year], head[year] + frame = pd.DataFrame( + { + "d": b["decile"], + "w": b["household_weight"], + "base": b["income"], + "change": h["income"] - b["income"], + } + ) + for row in impacts[year][:-1]: + group = frame[frame.d == row["decile"]] + if group.empty: + continue + changed = int(((group.change.abs() > 0.005) & (group.w > 0)).sum()) + if 0 < changed < 10: + assert row["suppressed"] and row["mean_change"] is None + continue + mean_change = (group.w * group.change).sum() / group.w.sum() + relative = (group.w * group.change).sum() / (group.w * group.base).sum() + # Published to what the comment shows: £1 and 0.1%. + assert row["mean_change"] == round(mean_change) + assert row["relative_change"] == round(relative, 3) + assert "households" not in row and "base_mean" not in row + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), shift=st.floats(-1e4, 1e4)) +def test_a_uniform_change_shows_in_every_decile(seed, shift): + rng = np.random.default_rng(seed) + base = households(200, rng, decile=np.repeat(np.arange(1, 11), 20)) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + shift + columns["household_id"] = columns["household_id"][::-1].copy() + for key in ("household_weight", "household_count_people", "decile", "income"): + columns[key] = columns[key][::-1].copy() + impacts, notes = decile_impacts(base, head, YEARS) + assert not any("differ" in n for n in notes) + for row in impacts[str(YEARS[0])]: + assert row["mean_change"] == pytest.approx(shift, abs=0.5 + 1e-6) + assert row["share_gaining"] == (1.0 if shift > 1 else 0.0) + assert row["share_losing"] == (1.0 if shift < -1 else 0.0) + + +@settings(max_examples=50, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(2, 300)) +def test_gini_matches_microdf(seed, n): + from microdf import MicroSeries + + rng = np.random.default_rng(seed) + income = rng.lognormal(10, 1, n) + weights = rng.uniform(0.5, 50, n) + assert weighted_gini(income, weights) == pytest.approx( + float(MicroSeries(income, weights=weights).gini()), abs=1e-6 + ) + + +def test_gini_bounds(): + assert weighted_gini(np.ones(10), np.ones(10)) == pytest.approx(0) + assert weighted_gini(np.r_[np.zeros(99), 1.0], np.ones(100)) == pytest.approx(0.99) + + +@pytest.mark.parametrize( + "value, kind, text", + [ + (1.234e9, "gbp", "+£1.23bn"), + (-5.5e6, "gbp", "-£5.5m"), + (0, "gbp", "0"), + (12_300, "count", "+12.3k"), + (0.0012, "rate", "+0.12pp"), + (None, "gbp", "n/a"), + ], +) +def test_change_formatting(value, kind, text): + assert fmt_change(value, kind) == text + + +def test_level_formatting(): + assert fmt_level(320.5e9, "gbp") == "£320.50bn" + assert fmt_level(0.2335, "rate") == "23.4%" + assert fmt_level(float("nan"), "gbp") == "n/a" + + +@settings(max_examples=20, deadline=None) +@given(values=metric_values, limit=st.integers(2000, 20000)) +def test_the_comment_carries_the_marker_and_fits_its_limit(values, limit): + head = {k: [v + 1e9 for v in series] for k, series in values.items()} + summary = compare(result(values), result(head), SPEC) + spec = SPEC.__class__(**{**SPEC.__dict__, "max_comment_characters": limit}) + text = render(summary, spec) + assert text.startswith(COMMENT_MARKER) + assert len(text) <= limit + 1 + assert render(summary, spec) == text + + +def test_a_skipped_head_passes_with_a_note(): + skipped = {"schema": TOOL_SCHEMA, "status": "skipped", "reason": "no data access"} + summary = compare(None, skipped, SPEC) + assert summary["status"] == "skipped" and summary["failures"] == [] + text = render(summary, SPEC) + assert text.startswith(COMMENT_MARKER) and "no data access" in text + + +def test_the_guard_section_lists_failures_and_mediated_responses(): + from policyengine_uk.utils.aggregate_impact.render import _guard + + guard = { + "status": "ran", + "year": 2026, + "components": [ + { + "variable": "incapacity_benefit", + "status": "fail", + "problems": [ + { + "message": "household_benefits: pension_credit rises in 2 households" + } + ], + "mediated": [ + { + "total": "household_benefits", + "head": "universal_credit", + "kind": "contains", + "households": 72, + "via": ["carers_allowance"], + } + ], + }, + {"variable": "vat", "status": "pass", "mediated": []}, + ], + "double_counts": [], + } + text = _guard({"head": {"guard": guard}}) + assert "held at its baseline value" in text + assert "1 failed; 0 were skipped" in text + assert "pension_credit rises in 2 households" in text + assert ( + "`universal_credit` responds only through other heads in 72 households " + "(other heads that moved there: `carers_allowance`" in text + ) + + +def test_a_figure_from_too_few_records_is_suppressed_end_to_end(): + """Sol re-review 2, finding 1: a total and caseload from one recipient + reveal the award and the weight, so neither is published.""" + from policyengine_uk.utils.aggregate_impact.api import to_report + from policyengine_uk.utils.aggregate_impact.measure import ( + Recorder, + _finite_metrics_check, + ) + from policyengine_uk.utils.aggregate_impact.render import _levels_table + + rec = Recorder() + rec.metric("spending.ben", "Ben", "spending", "gbp", 2025, 246.9, records=1) + rec.metric("spending.ben", "Ben", "spending", "gbp", 2026, 9e9, records=500) + rec.metric("spending.none", "None", "spending", "gbp", 2025, 0.0, records=0) + ben = rec.metrics["spending.ben"] + assert ben["values"] == {"2025": None, "2026": 9e9} + assert ben["suppressed"] == ["2025"] + # Zero records reveal nothing: an inert head stays at zero. + assert rec.metrics["spending.none"]["values"] == {"2025": 0.0} + _finite_metrics_check(rec) + assert rec.checks["finite:metrics"]["status"] == "pass" + + base = result({("receipts", "ben"): [1.0] * len(YEARS)}) + head = result({("receipts", "ben"): [1.0] * len(YEARS)}) + metric = next(m for m in head["metrics"] if m["id"] == "receipts.ben") + metric["values"][str(YEARS[0])] = None + metric["suppressed"] = [str(YEARS[0])] + summary = compare(base, head, SPEC) + assert summary["status"] == "pass", summary["failures"] + row = next(r for r in summary["rows"] if r["id"] == "receipts.ben") + assert row["changed"] and row["values"][str(YEARS[0])]["head_suppressed"] + assert summary["flags"] == [] + assert "suppressed" in _levels_table([row], YEARS) + assert "suppressed" in render(summary, SPEC) + report = to_report({**head, "partition": {"sha256": "p"}}) + first = next( + m + for m in report["metrics"] + if m["metric_id"] == "receipts.ben" and m["year"] == YEARS[0] + ) + assert first["value"] is None and first["suppressed"] is True + + +def test_published_guard_output_keeps_no_small_counts_or_shares(): + """Sol re-review 2, finding 3.""" + from policyengine_uk.utils.aggregate_impact.measure import publishable_guard + + guard = { + "status": "ran", + "components": [ + { + "variable": "x", + "units": 3, + "problems": [ + { + "households": 1, + "share": 0.125, + "message": "in fewer than 10 households", + }, + {"households": 40, "share": 0.5, "message": "in 40 households"}, + ], + "mediated": [{"households": 2, "head": "uc", "via": []}], + } + ], + } + guard["components"][0]["responses"] = {"gov_spending": {"uc": -0.55}} + guard["components"][0]["coefficients"] = {"gov_spending": 1.0} + out = publishable_guard(guard) + row = out["components"][0] + assert row["units"] is None + assert ( + row["problems"][0]["households"] is None and "share" not in row["problems"][0] + ) + # Sol re-review 3, finding 6: with only 3 perturbed units, no share and no + # response survives, even for a problem in 40 households. + assert row["problems"][1] == {"households": 40, "message": "in 40 households"} + assert row["responses"] == {} and row["coefficients"] == {} + assert row["mediated"][0]["households"] is None + # The input is not changed. + assert guard["components"][0]["units"] == 3 + # With enough perturbed units, a large problem keeps its share. + guard["components"][0]["units"] = 300 + kept = publishable_guard(guard)["components"][0] + assert kept["problems"][1]["share"] == 0.5 + assert kept["responses"] == {"gov_spending": {"uc": -0.55}} + + +# Sol re-review 3 of #2335: decile rows and protected totals' changes. + + +@pytest.mark.parametrize("changed", [1, 9, 10]) +def test_a_decile_row_from_fewer_than_10_changed_households_publishes_no_change( + changed, +): + rng = np.random.default_rng(0) + base = households(100, rng, decile=np.repeat(np.arange(1, 11), 10)) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"].copy() + columns["income"][:changed] += 250.0 # all in decile 1 + impacts, _ = decile_impacts(base, head, YEARS) + rows = {row["decile"]: row for row in impacts[str(YEARS[0])]} + assert rows[1]["suppressed"] == (changed < 10) + assert rows["all"]["suppressed"] == (changed < 10) + if changed < 10: + assert all(rows[1][k] is None for k in ("mean_change", "share_gaining")) + # Deciles where nothing changed publish a zero change. + assert rows[2]["mean_change"] == 0 and not rows[2]["suppressed"] + + +def test_a_zero_weight_household_does_not_count_as_changed(): + rng = np.random.default_rng(1) + weight = rng.uniform(1, 1000, 100) + weight[:10] = 0.0 # decile 1 carries no weight + base = households(100, rng, decile=np.repeat(np.arange(1, 11), 10), weight=weight) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"].copy() + columns["income"][:12] += 250.0 # 10 zero-weight in decile 1, 2 weighted in 2 + impacts, _ = decile_impacts(base, head, YEARS) + rows = {row["decile"]: row for row in impacts[str(YEARS[0])]} + # Twelve households change, but only two carry weight: a small cell. + assert rows[2]["suppressed"] and rows["all"]["suppressed"] + assert not rows[1]["suppressed"] + + +@settings(max_examples=50, deadline=None) +@given(seed=st.integers(0, 2**32 - 1)) +def test_published_decile_rows_carry_no_weights_and_display_precision_only(seed): + """The "all" row's mean change over its relative change, times the + household count, gave the exact base income total back.""" + rng = np.random.default_rng(seed) + base = households(300, rng) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + rng.normal(0, 500, 300) + impacts, _ = decile_impacts(base, head, YEARS) + for row in impacts[str(YEARS[0])]: + assert "households" not in row and "base_mean" not in row + if not row["suppressed"]: + assert row["mean_change"] == round(row["mean_change"]) + assert row["relative_change"] == round(row["relative_change"], 3) + assert row["share_gaining"] == round(row["share_gaining"], 3) + + +def _side(common, rare, records): + rec, _, _ = _rebuilt(common, rare, records=records) + return {m["id"]: m for m in rec.metrics.values()} + + +def _cell(rows, metric_id): + return next(r for r in rows if r["id"] == metric_id)["values"]["2026"] + + +def _changes(base, head): + from policyengine_uk.utils.aggregate_impact.compare import compare_metrics + + rows, flags = compare_metrics(base, head, SPEC, [2026]) + return rows, flags + + +def test_a_total_without_its_suppressed_head_changes_by_the_published_heads(): + """Sol re-review 5 of #2335: totals leave suppressed heads out, so their + exact change is the published heads' change, and is flagged normally.""" + rows, flags = _changes(_side(7.3e10, 246.9, 1), _side(7.4e10, 300.0, 1)) + total = _cell(rows, "model.gov_spending") + assert total["base"] == 7.3e10 and total["head"] == 7.4e10 + assert total["delta"] == pytest.approx(1e9) + assert total["excludes"] == ["spending.rare"] + assert any(f["id"] == "model.gov_spending" for f in flags) + # The suppressed head itself never shows a change. + assert _cell(rows, "spending.rare")["delta"] is None + # A total with no suppressed head is unchanged and marked nothing. + assert _cell(rows, "model.gov_tax")["excludes"] == [] + + +def test_a_suppression_transition_aligns_both_sides_from_published_values(): + """Sol re-review 4, finding 1: 10 recipients on the base and 9 on the head. + The base publishes the head, so it is taken out of the base's total too, + and the change is the common benefit's change alone.""" + rows, _ = _changes(_side(7.3e10, 246.9, 10), _side(7.4e10, 246.9, 9)) + total = _cell(rows, "model.gov_spending") + assert total["delta"] == pytest.approx(1e9) + assert total["excludes"] == ["spending.rare"] + balance = _cell(rows, "balance.balance") + assert balance["delta"] == pytest.approx(-1e9) + assert _cell(rows, "spending.rare")["delta"] is None + + +@settings(max_examples=200, deadline=None) +@given( + rare=st.floats(1.0, 4.9e8), + moved=st.floats(-1e8, 1e8), + common_change=st.floats(-1e10, 1e10), + base_records=st.sampled_from([1, 9, 10, 500]), + head_records=st.sampled_from([1, 9, 10, 500]), +) +def test_no_published_change_depends_on_a_suppressed_head( + rare, moved, common_change, base_records, head_records +): + """For all inputs: whatever a suppressed head does, on either side, every + published total and change equals what the published heads give.""" + base = _side(7.3e10, rare, base_records) + head = _side(7.3e10 + common_change, rare + moved, head_records) + rows, _ = _changes(base, head) + cells = {r["id"]: r["values"]["2026"] for r in rows} + rare_hidden = base_records < 10 or head_records < 10 + if rare_hidden: + assert cells["spending.rare"]["delta"] is None + # Totals change by the common head's change, nothing more. + for total in ( + "model.gov_spending", + "model.household_benefits", + "balance.spending", + ): + assert cells[total]["delta"] == pytest.approx( + common_change, rel=0, abs=0.01 + ) + # A side that suppresses the head publishes totals that don't depend on it. + for side, records, value in ( + (base, base_records, rare), + (head, head_records, rare + moved), + ): + if records < 10: + assert side["model.gov_spending"]["values"]["2026"] == pytest.approx( + side["spending.common"]["values"]["2026"], rel=0, abs=0.01 + ) + + +def test_a_suppressed_figure_and_a_marked_total_render_as_such(): + from policyengine_uk.utils.aggregate_impact.render import _change_cell, _deciles + + rows, _ = _changes(_side(7.3e10, 246.9, 1), _side(7.4e10, 300.0, 1)) + total = next(r for r in rows if r["id"] == "model.gov_spending") + assert _change_cell(total, 2026) == "+£1.00bn †" + rare = next(r for r in rows if r["id"] == "spending.rare") + assert _change_cell(rare, 2026) == "suppressed" + row = { + "decile": 1, + "suppressed": True, + "mean_change": None, + "relative_change": None, + "share_gaining": None, + "share_losing": None, + } + text = _deciles({"deciles": {"2026": [row, dict(row, decile="all")]}}, [2026]) + assert "fewer than 10 households change" in text and "n/a" not in text + + +def test_the_w4_format_lists_the_heads_a_total_leaves_out(): + side = _side(7.3e10, 246.9, 1) + result = { + "schema": "x", + "status": "ok", + "years": [2026], + "metrics": list(side.values()), + "checks": [], + "provenance": {}, + "partition": {}, + } + from policyengine_uk.utils.aggregate_impact.api import to_report + + rows = {(r["metric_id"], r["year"]): r for r in to_report(result)["metrics"]} + assert rows[("model.gov_spending", 2026)]["excludes"] == ["spending.rare"] + assert rows[("model.gov_tax", 2026)]["excludes"] == [] + # With their signs, for W4: a spending head enters the balances negative. + assert rows[("model.gov_spending", 2026)]["exclude_signs"] == {"spending.rare": 1} + assert rows[("balance.balance", 2026)]["exclude_signs"] == {"spending.rare": -1} + assert rows[("model.gov_tax", 2026)]["exclude_signs"] == {} + for row in rows.values(): + assert sorted(row["exclude_signs"]) == row["excludes"] + assert rows[("spending.rare", 2026)]["suppressed"] + + +def test_decile_changes_that_round_to_zero_are_not_called_no_change(): + """Sol re-review 4, finding 6.""" + from policyengine_uk.utils.aggregate_impact.render import _deciles + + rng = np.random.default_rng(3) + base = households(100, rng, decile=np.repeat(np.arange(1, 11), 10)) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + 0.25 + impacts, _ = decile_impacts(base, head, YEARS) + assert all( + r["mean_change"] == 0 and r["any_change"] for r in impacts[str(YEARS[0])] + ) + text = _deciles({"deciles": {str(YEARS[0]): impacts[str(YEARS[0])]}}, [YEARS[0]]) + assert "rounds to £0" in text and "No household" not in text + unchanged, _ = decile_impacts(base, copy.deepcopy(base), YEARS) + text = _deciles({"deciles": {str(YEARS[0]): unchanged[str(YEARS[0])]}}, [YEARS[0]]) + assert "No household's net income changes" in text + + +def test_a_mean_or_clipped_total_with_different_exclusions_is_withheld(): + """Sol re-review 6 of #2335, P2 5: subtracting a national head from a + per-household mean or a total reset at zero gives a wrong change. Sol + re-review 7, P3 5: the withheld change was then called "no change".""" + from policyengine_uk.utils.aggregate_impact.compare import compare_metrics + from policyengine_uk.utils.aggregate_impact.render import ( + _change_cell, + _changes_table, + ) + + def side(value, excludes): + record = { + "id": "deciles.mean_income.1", + "label": "d1", + "section": "deciles", + "kind": "gbp_household", + "values": {"2026": value}, + } + if excludes: + record["excludes"] = {"2026": excludes} + rare = { + "id": "spending.rare", + "label": "R", + "section": "spending", + "kind": "gbp", + "values": {"2026": None if excludes else 1_000.0}, + **({"suppressed": ["2026"]} if excludes else {}), + } + return {"deciles.mean_income.1": record, "spending.rare": rare} + + rows, _ = compare_metrics( + side(1_100.0, None), side(1_000.0, {"spending.rare": 1}), SPEC, [2026] + ) + cell = _cell(rows, "deciles.mean_income.1") + assert cell["delta"] is None and cell["withheld"] + row = next(r for r in rows if r["id"] == "deciles.mean_income.1") + assert row["changed"] + assert _change_cell(row, 2026) == "withheld †" + assert "d1" in _changes_table(rows, [2026]) + # Equal on both sides, with equal exclusions: a real "no change". + rows, _ = compare_metrics( + side(1_000.0, {"spending.rare": 1}), + side(1_000.0, {"spending.rare": 1}), + SPEC, + [2026], + ) + cell = _cell(rows, "deciles.mean_income.1") + assert cell["delta"] == 0 and not cell["withheld"] + assert not next(r for r in rows if r["id"] == "deciles.mean_income.1")["changed"] + + +def test_a_figure_suppressed_on_both_sides_is_not_called_unchanged(): + rows, _ = _changes(_side(7.3e10, 246.9, 1), _side(7.3e10, 246.9, 1)) + rare = next(r for r in rows if r["id"] == "spending.rare") + assert rare["values"]["2026"]["withheld"] and rare["changed"] + # The total holds only the common head, which didn't change. + total = next(r for r in rows if r["id"] == "model.gov_spending") + assert total["values"]["2026"]["delta"] == 0 and not total["changed"] + + +def test_decile_rows_are_withheld_when_the_sides_exclude_different_heads(): + """Sol re-review 6 of #2335, P2 6.""" + rng = np.random.default_rng(5) + base = households(100, rng, decile=np.repeat(np.arange(1, 11), 10)) + head = copy.deepcopy(base) + for year, columns in base.items(): + columns["excludes"] = np.array(["spending.rare"], dtype=str) + head[year]["excludes"] = np.array([], dtype=str) + impacts, notes = decile_impacts(base, head, YEARS) + assert impacts == {} + assert any("suppressed on one side only" in n for n in notes) + + +def test_a_year_ranked_on_a_suppressed_figure_is_withheld_with_a_reason(): + rng = np.random.default_rng(7) + base = households(100, rng, decile=np.repeat(np.arange(1, 11), 10)) + head = copy.deepcopy(base) + impacts, notes = decile_impacts(base, head, YEARS, withheld=[YEARS[0]]) + assert str(YEARS[0]) not in impacts and str(YEARS[1]) in impacts + assert any("ranked on an income holding" in n for n in notes) + + +def test_a_comparison_withheld_in_every_year_says_why(): + """Sol re-review 8 of #2335, P3 1: with every year withheld, measure keeps + no household detail, and the comment said it was missing.""" + from policyengine_uk.utils.aggregate_impact.render import render + + base = result({("receipts", "a"): [1.0] * len(YEARS)}) + head = copy.deepcopy(base) + head["metrics"].append( + { + "id": "deciles.mean_income.1", + "label": "d1", + "section": "deciles", + "group": None, + "kind": "gbp_household", + "unit": None, + "values": {str(y): None for y in YEARS}, + "suppressed": [str(y) for y in YEARS], + } + ) + summary = compare(base, head, SPEC) + assert summary["deciles"] == {} + assert len(summary["decile_notes"]) == len(YEARS) + assert all("ranked on an income holding" in n for n in summary["decile_notes"]) + text = render(summary, SPEC) + assert "need household detail" not in text and "ranked on an income" in text + # Without any withheld year, missing detail is still called missing. + summary = compare(base, copy.deepcopy(base), SPEC) + assert summary["decile_notes"] == [] + assert "need household detail" in render(summary, SPEC) + + +def test_withheld_decile_years_are_explained_not_called_missing(): + """Sol re-review 7 of #2335, P3 5: with every year withheld, the comment + dropped the notes and said household detail was missing.""" + from policyengine_uk.utils.aggregate_impact.render import _deciles + + note = "2026-27: a figure from fewer than 10 records is suppressed" + text = _deciles({"deciles": {}, "decile_notes": [note]}, [2026]) + assert note in text and "need household detail" not in text + text = _deciles({"deciles": {}, "decile_notes": []}, [2026]) + assert "need household detail" in text diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py new file mode 100644 index 0000000000..5268196c19 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py @@ -0,0 +1,90 @@ +"""Mandatory-data mode end to end, in a subprocess with this repo's hooks. + +A scratch test directory imports the real conftest's hooks, so the cases +exercise the hook ordering pytest actually applies. +""" + +import os +import subprocess +import sys +import textwrap + +import pytest + +CONFTEST = "from policyengine_uk.tests.conftest import * # noqa: F401,F403\n" +CASES = textwrap.dedent( + """ + import pytest + + def test_needs_no_data(): + assert True + + @pytest.mark.microsimulation + def test_needs_data(): + assert True + + @pytest.mark.microsimulation + @pytest.mark.skip(reason="a skip marker must not hide missing data") + def test_needs_data_but_is_marked_skip(): + assert True + """ +) + + +def run(tmp_path, **env_changes): + (tmp_path / "conftest.py").write_text(CONFTEST) + (tmp_path / "test_cases.py").write_text(CASES) + env = { + k: v + for k, v in os.environ.items() + if k + not in ( + "HUGGING_FACE_TOKEN", + "POLICYENGINE_UK_DEFAULT_DATASET", + "POLICYENGINE_UK_REQUIRE_TEST_DATA", + "PYTEST_ADDOPTS", + ) + } + env.update(env_changes) + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(tmp_path), + "-q", + "-p", + "no:cacheprovider", + "-p", + "no:xdist", + "--rootdir", + str(tmp_path), + "-W", + "ignore::pytest.PytestUnknownMarkWarning", + ], + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +def test_without_data_the_data_tests_skip_and_the_rest_run(tmp_path): + done = run(tmp_path) + assert done.returncode == 0, done.stdout + done.stderr + assert "1 passed, 2 skipped" in done.stdout + + +def test_in_mandatory_mode_every_data_test_fails_even_one_marked_skip(tmp_path): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA="1") + assert done.returncode != 0 + assert "1 passed, 2 errors" in done.stdout, done.stdout + assert "skipped" not in done.stdout.splitlines()[-1] + assert "Mandatory-data mode" in done.stdout + + +@pytest.mark.parametrize("value", ["maybe", "2"]) +def test_an_unrecognised_mode_value_fails_closed(tmp_path, value): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA=value) + assert done.returncode != 0 + assert "2 errors" in done.stdout, done.stdout diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py new file mode 100644 index 0000000000..397fcfecc2 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py @@ -0,0 +1,191 @@ +"""The certified dataset pin, the fixtures that record it, and the data mode.""" + +import hashlib +import os +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.tests.conftest import ( + CERTIFIED_DATASET_PIN, + DATASET_ENV_VAR, + DEFAULT_TEST_DATASET_URL, + REQUIRE_DATA_ENV_VAR, + TOKEN_ENV_VAR, + plan_test_data, +) +from policyengine_uk.utils.aggregate_impact.dataset import ( + DatasetMismatchError, + DatasetPinError, + load_pin, + sha256_file, + verify_file, +) + +TESTS = Path(__file__).resolve().parents[1] +PIN = load_pin(CERTIFIED_DATASET_PIN) + + +def test_pin_is_immutable_and_complete(): + assert PIN.problems() == [] + assert len(PIN.revision) == 40 and len(PIN.sha256) == 64 + assert PIN.uri == f"hf://{PIN.repo_id}/{PIN.path}@{PIN.revision}" + assert DEFAULT_TEST_DATASET_URL == PIN.uri + + +def test_reform_fixtures_record_the_pinned_dataset(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + recorded = fixtures["dataset"] + assert recorded["revision"] == PIN.revision + assert recorded["sha256"] == PIN.sha256 + assert recorded["path"] == PIN.path + assert str(recorded["data_version"]) == PIN.data_version + + +def test_every_reform_fixture_keeps_its_receipts(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + for reform in fixtures["reforms"]: + receipts = reform["receipts"] + assert len(receipts) >= 2, reform["name"] + latest = receipts[-1] + assert PIN.data_version in latest["data"], reform["name"] + assert round(latest["impact"], 1) == reform["expected_impact"], reform["name"] + + +def _pin_file(tmp_path, **overrides): + raw = { + "repo_id": PIN.repo_id, + "path": PIN.path, + "revision": PIN.revision, + "sha256": PIN.sha256, + **overrides, + } + path = tmp_path / "pin.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +@pytest.mark.parametrize( + "overrides", + [ + {"revision": "1.58.0"}, + {"revision": "main"}, + {"sha256": "5BA399EA"}, + {"repo_id": "no-slash"}, + {"path": "/absolute.h5"}, + {"unexpected": "key"}, + ], +) +def test_mutable_or_malformed_pins_are_rejected(tmp_path, overrides): + with pytest.raises(DatasetPinError): + load_pin(_pin_file(tmp_path, **overrides)) + + +@settings(max_examples=50, deadline=None) +@given(revision=st.text(max_size=45)) +def test_only_a_full_commit_hash_is_accepted_as_a_revision(revision): + from dataclasses import replace + + pin = replace(PIN, revision=revision) + is_commit = len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + assert (not any("revision" in p for p in pin.problems())) == is_commit + + +@settings(max_examples=30, deadline=None) +@given(content=st.binary(max_size=4096), tamper=st.integers(0, 4095)) +def test_verification_accepts_the_pinned_bytes_and_rejects_any_other( + tmp_path_factory, content, tamper +): + path = tmp_path_factory.mktemp("data") / "file.h5" + path.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + assert sha256_file(path) == digest + assert verify_file(path, digest) == digest + if content: + changed = bytearray(content) + changed[tamper % len(content)] ^= 0xFF + path.write_bytes(bytes(changed)) + with pytest.raises(DatasetMismatchError): + verify_file(path, digest) + + +ENV_VALUES = st.one_of( + st.none(), st.sampled_from(["", "0", "1", "true", "FALSE", "yes", "maybe"]) +) +DATASETS = st.one_of( + st.none(), st.just(""), st.just(PIN.uri), st.just("hf://other/repo/file.h5@abc") +) +TOKENS = st.one_of(st.none(), st.just(""), st.just("hf_token")) + + +def _env(flag, dataset, token, extra): + env = dict(extra) + for key, value in ( + (REQUIRE_DATA_ENV_VAR, flag), + (DATASET_ENV_VAR, dataset), + (TOKEN_ENV_VAR, token), + ): + if value is not None: + env[key] = value + return env + + +@settings(max_examples=300, deadline=None) +@given( + flag=ENV_VALUES, + dataset=DATASETS, + token=TOKENS, + extra=st.dictionaries(st.sampled_from(["HOME", "CI", "PATH"]), st.text(max_size=5)), +) +def test_data_plan_invariants(flag, dataset, token, extra): + plan = plan_test_data(_env(flag, dataset, token, extra), PIN.uri) + assert plan.action in ("run", "skip", "fail") + # Mandatory mode never skips, and runs only on the pinned dataset. + if plan.mandatory: + assert plan.action != "skip" + if plan.action == "run": + assert plan.pinned and plan.dataset == PIN.uri + # Running always names a dataset; an explicit one always wins. + if plan.action == "run": + assert plan.dataset + if dataset: + assert plan.dataset == dataset + # Nothing configured and not mandatory: the old behaviour, a skip. + if ( + not dataset + and not token + and str(flag or "").strip().lower() in ("", "0", "false", "no") + ): + assert plan.action == "skip" + # An unrecognised mode value fails closed. + if flag is not None and flag.strip().lower() not in ( + "", + "0", + "1", + "true", + "false", + "yes", + "no", + "on", + "off", + ): + assert plan.action == "fail" + # Unrelated variables never change the plan. + assert plan == plan_test_data(_env(flag, dataset, token, {}), PIN.uri) + + +def test_data_plan_examples(): + assert plan_test_data({TOKEN_ENV_VAR: "t"}, PIN.uri).action == "run" + assert plan_test_data({}, PIN.uri).action == "skip" + assert plan_test_data({REQUIRE_DATA_ENV_VAR: "1"}, PIN.uri).action == "fail" + assert plan_test_data( + {REQUIRE_DATA_ENV_VAR: "1", TOKEN_ENV_VAR: "t"}, PIN.uri + ).pinned + other = {REQUIRE_DATA_ENV_VAR: "1", DATASET_ENV_VAR: "local.h5"} + assert plan_test_data(other, PIN.uri).action == "fail" + assert plan_test_data({DATASET_ENV_VAR: "local.h5"}, PIN.uri).action == "run" diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py new file mode 100644 index 0000000000..7d801764b1 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py @@ -0,0 +1,504 @@ +"""The counted-once check, on a small fake simulation whose formulas are known. + +The fake implements only the simulation API the check uses, so each case +controls exactly which component contains or reads which. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.guard import counted_once, snapshot_inputs +from policyengine_uk.utils.aggregate_impact.partition import parse_partition + +YEAR = 2026 + + +class FakeHolder: + def __init__(self, sim, variable): + self.sim, self.variable = sim, variable + + def get_known_periods(self): + return [p for (v, p) in self.sim.inputs if v == self.variable] + + def get_array(self, period): + return self.sim.inputs[(self.variable, str(period))] + + +class FakeSim: + """Household-level variables only; formulas read each other by name.""" + + def __init__(self, formulas, inputs, weights): + self.formulas = formulas + self.inputs = { + (k, str(YEAR)): np.asarray(v, dtype=float) for k, v in inputs.items() + } + self.inputs[("household_weight", str(YEAR))] = np.asarray(weights, dtype=float) + self.cache = {} + names = set(formulas) | set(inputs) | {"household_weight"} + self.tax_benefit_system = SimpleNamespace( + variables={ + name: SimpleNamespace( + entity=SimpleNamespace(key="household"), + definition_period="year", + value_type=float, + ) + for name in names + } + ) + + def calculate(self, variable, period, unweighted=True, map_to=None): + key = (variable, str(period)) + if key in self.inputs: + return self.inputs[key] + if key not in self.cache: + self.cache[key] = np.asarray( + self.formulas[variable](lambda name: self.calculate(name, period)), + dtype=float, + ) + return self.cache[key] + + def map_result(self, values, source, target): + assert source == target == "household" + return values + + def set_input(self, variable, period, values): + self.inputs[(variable, str(period))] = np.asarray(values, dtype=float) + + def delete_arrays(self, variable, period=None): + for store in (self.inputs, self.cache): + for key in [k for k in store if k[0] == variable]: + del store[key] + + def get_holder(self, variable): + return FakeHolder(self, variable) + + def _invalidate_all_caches(self): + self.cache = {} + + +def partition(tax_heads, spending_heads=(), double_counts=(), reads=None): + reads = reads or {} + + def heads(spec, total): + return [ + { + "id": v, + "label": v, + "variable": v, + "totals": [total], + "entity": "household", + "reads": [{"head": r} for r in reads.get(v, [])], + } + for v in spec + ] + + return parse_partition( + { + "schema_version": 1, + "totals": {"gov_tax": {}, "gov_spending": {}}, + "tax_heads": heads(tax_heads, "gov_tax"), + "spending_heads": heads(spending_heads, "gov_spending"), + "model_double_counts": [ + {"variable": v, "contained_in": c, "totals": ["gov_tax"]} + for v, c in double_counts + ], + } + ) + + +def model(contains=True, interaction=-0.5): + """tax_a may contain tax_b; ben_d reads ben_c with the given slope.""" + return { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "tax_a": lambda f: 0.1 * f("x") + (f("tax_b") if contains else 0), + "gov_tax": lambda f: f("tax_a") + f("tax_b"), + "ben_c": lambda f: 100 + 0 * f("x"), + "ben_d": lambda f: np.maximum(0, 200 + interaction * f("ben_c")), + "gov_spending": lambda f: f("ben_c") + f("ben_d"), + } + + +def run(formulas, part, x=(1000, 2000, 3000), weights=(1.5, 2.0, 0.5)): + sim = FakeSim(formulas, {"x": x}, weights) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + return sim, result + + +def by_variable(result): + return {r["variable"]: r for r in result["components"]} + + +def kinds(row): + """{(head, kind)} of a component's problems.""" + return {(p["head"], p["kind"]) for p in row.get("problems") or []} + + +def test_declared_double_count_passes_and_the_model_total_counts_it_twice(): + _, result = run( + model(contains=True), + partition(["tax_a"], ["ben_c", "ben_d"], [("tax_b", "tax_a")]), + ) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" + assert rows["tax_b"]["coefficients"]["gov_tax"] == pytest.approx(1) + assert rows["tax_b"]["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert result["double_counts"][0]["status"] == "pass" + assert result["reset_reproduces_baseline"] + + +def test_stale_double_count_declaration_fails(): + _, result = run( + model(contains=False), partition(["tax_a"], [], [("tax_b", "tax_a")]) + ) + assert result["double_counts"][0]["status"] == "fail" + assert result["double_counts"][0]["coefficient"] == pytest.approx(0) + + +def test_undeclared_containment_fails_because_another_head_moves(): + _, result = run(model(contains=True), partition(["tax_a", "tax_b"])) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "fail" + assert ("tax_a", "moves") in kinds(rows["tax_b"]) + assert rows["tax_b"]["responses"]["gov_tax"]["tax_a"] == pytest.approx(1) + assert rows["tax_a"]["status"] == "pass" + + +def test_means_tested_offset_passes_but_a_benefit_rising_with_income_fails(): + _, offset = run(model(interaction=-0.5), partition(["tax_a"], ["ben_c", "ben_d"])) + row = by_variable(offset)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(0.5) + assert row["status"] == "pass" + _, rising = run(model(interaction=0.2), partition(["tax_a"], ["ben_c", "ben_d"])) + row = by_variable(rising)["ben_c"] + assert row["status"] == "fail" and ("ben_d", "rises") in kinds(row) + + +def test_a_total_that_adds_a_zero_component_twice_fails(): + """Baseline identities hold while tax_b is zero; the response gives it away.""" + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["gov_tax"] = lambda f: f("tax_a") + 2 * f("tax_b") + _, result = run(formulas, partition(["tax_a", "tax_b"])) + row = by_variable(result)["tax_b"] + assert row["coefficients"]["gov_tax"] == pytest.approx(1) + assert row["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert row["status"] == "fail" and ("gov_tax", "model_total") in kinds(row) + + +def test_a_means_test_fall_cannot_hide_a_benefit_that_contains_another(): + """ben_e contains ben_c while ben_d falls pound for pound: the net is 1.""" + formulas = model(interaction=-1.0) + formulas["ben_e"] = lambda f: 50 + f("ben_c") + formulas["gov_spending"] = lambda f: f("ben_c") + f("ben_d") + f("ben_e") + _, result = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e"])) + row = by_variable(result)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(1) + assert row["status"] == "fail" and ("ben_e", "rises") in kinds(row) + + +def test_an_eligibility_loss_in_the_same_household_cannot_hide_a_double_count(): + """One household's ben_e contains ben_c and it loses £100 of ben_f at once.""" + formulas = model(interaction=0.0) + first = np.array([1.0, 0.0, 0.0]) + formulas["ben_e"] = lambda f: 50 + first * f("ben_c") + formulas["ben_f"] = lambda f: np.where( + (f("ben_c") > 100.5) & (first == 1), 0.0, 100.0 + ) + formulas["gov_spending"] = lambda f: ( + f("ben_c") + f("ben_d") + f("ben_e") + f("ben_f") + ) + _, result = run( + formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e", "ben_f"]) + ) + row = by_variable(result)["ben_c"] + assert row["status"] == "fail" and ("ben_e", "rises") in kinds(row) + + +def test_a_declared_read_may_respond_but_not_one_for_one(): + formulas = model(contains=False) + formulas["tax_a"] = lambda f: 0.1 * f("x") + 0.3 * f("tax_b") + reading = partition(["tax_a", "tax_b"], reads={"tax_a": ["tax_b"]}) + _, result = run(formulas, reading) + assert by_variable(result)["tax_b"]["status"] == "pass" + _, undeclared = run(formulas, partition(["tax_a", "tax_b"])) + assert by_variable(undeclared)["tax_b"]["status"] == "fail" + _, containing = run(model(contains=True), reading) + row = by_variable(containing)["tax_b"] + assert row["status"] == "fail" and ("tax_a", "contains") in kinds(row) + + +def test_supplied_inputs_are_restored_after_perturbation(): + formulas = model(contains=False) + formulas.pop("tax_b") + sim = FakeSim(formulas, {"x": (1, 2, 3), "tax_b": (7, 0, 9)}, (1, 1, 1)) + part = partition(["tax_a", "tax_b"]) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + assert np.array_equal(sim.inputs[("tax_b", str(YEAR))], [7, 0, 9]) + assert result["reset_reproduces_baseline"] + assert by_variable(result)["tax_b"]["units"] == 2 + + +def test_zero_components_are_perturbed_everywhere(): + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["ben_c"] = lambda f: 0 * f("x") + _, result = run(formulas, partition(["tax_a", "tax_b"], ["ben_c", "ben_d"])) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" and rows["tax_b"]["units"] == 3 + assert rows["ben_c"]["status"] == "pass" and rows["ben_c"]["units"] == 3 + + +def test_a_formula_result_computed_earlier_is_not_frozen_as_an_input(): + """On real policyengine-core, a holder's known periods include results.""" + from policyengine_core.country_template import CountryTaxBenefitSystem + from policyengine_core.simulations import SimulationBuilder + + system = CountryTaxBenefitSystem() + sim = SimulationBuilder().build_from_dict( + system, {"persons": {"a": {"salary": {"2026-01": 3000}}}} + ) + sim.calculate("income_tax", "2026-01") + assert sim.get_holder("income_tax").get_known_periods() + snapshot = snapshot_inputs(sim, ["salary", "income_tax"]) + assert set(snapshot) == {"salary"} + + +@settings(max_examples=40, deadline=None) +@given( + x=st.lists(st.floats(1, 1e6), min_size=1, max_size=8), + weight=st.floats(0.1, 1e4), + contains=st.booleans(), +) +def test_the_model_coefficient_is_two_exactly_when_a_head_contains_another( + x, weight, contains +): + _, result = run( + model(contains=contains), + partition(["tax_a", "tax_b"]), + x=x, + weights=[weight] * len(x), + ) + row = by_variable(result)["tax_b"] + assert row["model_coefficients"]["gov_tax"] == pytest.approx( + 2 if contains else 1, rel=1e-6 + ) + assert (row["status"] == "fail") == contains + assert result["reset_reproduces_baseline"] + + +def test_a_reader_that_contains_the_component_in_some_households_fails(): + """Containment in one household of three: the aggregate is far from 1.""" + formulas = model(contains=False) + first = np.array([1.0, 0.0, 0.0]) + formulas["tax_a"] = lambda f: ( + 0.1 * f("x") + first * f("tax_b") - 0.3 * (1 - first) * f("tax_b") + ) + _, result = run(formulas, partition(["tax_a", "tax_b"], reads={"tax_a": ["tax_b"]})) + row = by_variable(result)["tax_b"] + assert row["status"] == "fail" and ("tax_a", "contains") in kinds(row) + assert ( + next(p for p in row["problems"] if p["kind"] == "contains")["households"] == 1 + ) + + +def test_a_passport_on_a_zero_benefit_passes_but_containment_with_it_fails(): + """ben_z is zero for everyone; ben_d pays £40 to anyone receiving it.""" + formulas = model(interaction=0.0) + formulas["ben_z"] = lambda f: 0 * f("x") + formulas["ben_d"] = lambda f: 200 + 40 * (f("ben_z") > 0) + formulas["gov_spending"] = lambda f: f("ben_c") + f("ben_d") + f("ben_z") + _, passport = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_z"])) + assert by_variable(passport)["ben_z"]["status"] == "pass" + formulas["ben_d"] = lambda f: 200 + 40 * (f("ben_z") > 0) + f("ben_z") + _, containing = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_z"])) + row = by_variable(containing)["ben_z"] + assert row["status"] == "fail" and ("ben_d", "contains") in kinds(row) + + +def test_a_head_containing_the_component_for_one_small_household_fails(): + """Judged household by household, a tiny share cannot pass as noise.""" + formulas = model(contains=False) + one = np.array([0.0, 0.0, 1.0]) + formulas["tax_a"] = lambda f: 0.1 * f("x") + one * f("tax_b") + _, result = run(formulas, partition(["tax_a", "tax_b"]), weights=(1e6, 1e6, 1e-3)) + row = by_variable(result)["tax_b"] + assert row["status"] == "fail" and ("tax_a", "moves") in kinds(row) + + +def test_a_component_only_in_zero_weight_households_is_skipped(): + formulas = model(contains=False) + formulas["tax_b"] = lambda f: np.array([5.0, 0.0, 0.0]) + 0 * f("x") + _, result = run(formulas, partition(["tax_a", "tax_b"]), weights=(0.0, 1.0, 1.0)) + row = by_variable(result)["tax_b"] + assert row["status"] == "skipped" and "no weight" in row["detail"] + + +def test_a_taper_that_adds_the_component_back_is_beyond_the_check(): + """Documented limit: max(0, a - c) + c does not move at the margin.""" + formulas = model(interaction=0.0) + formulas["ben_g"] = lambda f: np.maximum(0, 300 - f("ben_c")) + f("ben_c") + formulas["gov_spending"] = lambda f: f("ben_c") + f("ben_d") + f("ben_g") + _, result = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_g"])) + assert by_variable(result)["ben_c"]["status"] == "pass" + + +def overlap_model(uc_reads_ca=1.0, uc_contains_ib=0.0, ib=10.0): + """ca is reduced by ib (overlapping benefits); uc counts ca as income. + + The model's pattern of Incapacity Benefit, Carer's Allowance and + Universal Credit (#2424): raising ib lowers ca, and uc rises by what ca + lost. + """ + formulas = model(interaction=0.0) + formulas["ib"] = lambda f: ib + 0 * f("x") + formulas["ca"] = lambda f: np.maximum(0, 100 - f("ib")) + formulas["uc"] = lambda f: np.maximum( + 0, 500 - uc_reads_ca * f("ca") + uc_contains_ib * f("ib") + ) + formulas["gov_spending"] = lambda f: f("ib") + f("ca") + f("uc") + return formulas + + +OVERLAP = partition(["tax_a"], ["ib", "ca", "uc"]) + + +def test_a_benefit_that_rises_through_another_head_is_mediated_not_contained(): + sim, result = run(overlap_model(), OVERLAP) + row = by_variable(result)["ib"] + assert row["status"] == "pass" + assert row["mediated"] == [ + { + "total": "gov_spending", + "head": "uc", + "kind": "rises", + "households": 3, + "via": ["ca"], + } + ] + assert result["reset_reproduces_baseline"] + # Holding heads leaves no input behind. + assert {k[0] for k in sim.inputs} == {"x", "household_weight"} + + +def test_a_mediated_route_cannot_hide_a_direct_containment(): + _, result = run(overlap_model(uc_contains_ib=1.0), OVERLAP) + row = by_variable(result)["ib"] + assert row["status"] == "fail" and ("uc", "rises") in kinds(row) + assert row["mediated"] == [] + + +def test_a_zero_benefit_mediated_one_for_one_is_not_containment(): + """ib is zero for everyone, so uc may respond; its slope of 1 is via ca.""" + _, result = run(overlap_model(ib=0.0), OVERLAP) + row = by_variable(result)["ib"] + assert row["status"] == "pass" + assert [(m["head"], m["kind"], m["via"]) for m in row["mediated"]] == [ + ("uc", "contains", ["ca"]) + ] + _, containing = run(overlap_model(ib=0.0, uc_contains_ib=1.0), OVERLAP) + row = by_variable(containing)["ib"] + assert row["status"] == "fail" and ("uc", "contains") in kinds(row) + + +def test_a_tax_moved_by_a_declared_reader_is_mediated_and_found_directly(): + """tax_c reads tax_b (declared); tax_a reads tax_c (undeclared).""" + formulas = model(contains=False) + formulas["tax_c"] = lambda f: 0.2 * f("tax_b") + 0 * f("x") + formulas["tax_a"] = lambda f: 0.1 * f("x") + 0.5 * f("tax_c") + formulas["gov_tax"] = lambda f: f("tax_a") + f("tax_b") + f("tax_c") + _, result = run( + formulas, partition(["tax_a", "tax_b", "tax_c"], reads={"tax_c": ["tax_b"]}) + ) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" + assert [(m["head"], m["via"]) for m in rows["tax_b"]["mediated"]] == [ + ("tax_a", ["tax_c"]) + ] + assert rows["tax_c"]["status"] == "fail" + assert ("tax_a", "moves") in kinds(rows["tax_c"]) + + +@settings(max_examples=40, deadline=None) +@given( + reads_ca=st.floats(0, 1), + contains_ib=st.sampled_from([0.0, 0.25, 1.0]), + ib=st.sampled_from([0.0, 10.0]), +) +def test_the_guard_fails_exactly_when_a_head_responds_directly( + reads_ca, contains_ib, ib +): + """Whatever the mediated route carries, only the direct response fails.""" + _, result = run(overlap_model(reads_ca, contains_ib, ib), OVERLAP) + row = by_variable(result)["ib"] + # With ib zero for everyone uc may respond, except one for one. + direct = contains_ib == 1.0 if ib == 0.0 else contains_ib > 0 + assert (row["status"] == "fail") == direct + assert result["reset_reproduces_baseline"] + + +def test_a_rise_cancelled_everywhere_by_a_mediated_fall_is_beyond_the_check(): + """Documented limit: uc contains ib but falls with ca, which ib lowers.""" + _, hidden = run(overlap_model(uc_reads_ca=-1.0, uc_contains_ib=1.0), OVERLAP) + assert by_variable(hidden)["ib"]["status"] == "pass" + # In a household that has no Carer's Allowance the rise shows. + formulas = overlap_model(uc_reads_ca=-1.0, uc_contains_ib=1.0) + carer = np.array([1.0, 1.0, 0.0]) + formulas["ca"] = lambda f: carer * np.maximum(0, 100 - f("ib")) + _, shown = run(formulas, OVERLAP) + row = by_variable(shown)["ib"] + assert row["status"] == "fail" and ("uc", "rises") in kinds(row) + assert next(p for p in row["problems"] if p["head"] == "uc")["households"] == 1 + + +def test_a_containment_gated_by_a_passport_on_another_head_fails(): + """Sol re-review 2, finding 2: c is zero for everyone, b passports on c, + and h contains c only where b is paid. Holding b at its baseline (zero) + switches h off; holding it at its perturbed value keeps h's slope of 1.""" + formulas = model(interaction=0.0) + formulas["c"] = lambda f: 0 * f("x") + formulas["b"] = lambda f: 40.0 * (f("c") > 0) + formulas["h"] = lambda f: f("c") * (f("b") > 0) + formulas["gov_spending"] = lambda f: ( + f("c") + f("b") + f("h") + f("ben_c") + f("ben_d") + ) + _, result = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "c", "b", "h"])) + row = by_variable(result)["c"] + assert row["status"] == "fail" and ("h", "contains") in kinds(row) + assert result["reset_reproduces_baseline"] + + +def test_a_mediated_fall_that_outweighs_a_containment_is_beyond_the_check(): + """Documented limit: uc contains ib but falls by twice what ca lost.""" + _, result = run(overlap_model(uc_reads_ca=-2.0, uc_contains_ib=1.0), OVERLAP) + assert by_variable(result)["ib"]["status"] == "pass" + + +def test_mediated_records_name_the_heads_that_moved_not_a_proven_route(): + """Sol re-review 2, finding 4: an unrelated head that moved is listed too.""" + formulas = overlap_model() + formulas["unrelated"] = lambda f: 200 - 0.5 * f("ib") + formulas["gov_spending"] = lambda f: f("ib") + f("ca") + f("uc") + f("unrelated") + _, result = run(formulas, partition(["tax_a"], ["ib", "ca", "uc", "unrelated"])) + row = by_variable(result)["ib"] + assert row["status"] == "pass" + assert row["mediated"][0]["via"] == ["ca", "unrelated"] + + +def test_units_and_households_count_only_records_with_weight(): + """Sol re-review 3 of #2335, finding 4: a zero-weight record adds nothing.""" + formulas = model(contains=False) + one = np.array([1.0, 1.0, 0.0]) + formulas["tax_a"] = lambda f: 0.1 * f("x") + one * f("tax_b") + _, result = run(formulas, partition(["tax_a", "tax_b"]), weights=(1.5, 0.0, 0.5)) + row = by_variable(result)["tax_b"] + # tax_b is non-zero in all three households, one of which has no weight. + assert row["units"] == 2 + # tax_a contains tax_b in households 1 and 2; only household 1 has weight. + moves = [p for p in row["problems"] if p["head"] == "tax_a"] + assert moves and all(p["households"] == 1 for p in moves) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py new file mode 100644 index 0000000000..f4ba983978 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py @@ -0,0 +1,1058 @@ +"""Measurement bookkeeping: check scopes, reform costing checks, digests.""" + +import dataclasses +from types import SimpleNamespace + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.api import to_report +from policyengine_uk.utils.aggregate_impact.compare import classify_check +from policyengine_uk.utils.aggregate_impact.measure import ( + MIN_CELL, + cell_detail, + Recorder, + _finite_metrics_check, + _digest_tool_files, + definitions_digest, + measure_reforms, + tool_digest, +) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parse_reforms +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +SPEC = load_spec() +YEARS = [2025, 2026] + + +# Households in the stub worlds: a change in fewer than 10 is a small cell. +N = 20 +NO_HEADS = SimpleNamespace(heads=(), double_counts=()) + + +def _baseline(sim, years, targets=()): + """The baseline records a reform is checked against, with nothing + suppressed.""" + from policyengine_uk.utils.aggregate_impact.measure import baseline_records + + return {y: baseline_records(sim, NO_HEADS, Recorder(), y, targets) for y in years} + + +class YearSim: + """A simulation whose gov_balance per household depends on the year.""" + + def __init__(self, balance_by_year): + self.balance = balance_by_year + entity = SimpleNamespace(key="household") + self.tax_benefit_system = SimpleNamespace( + variables={ + "gov_balance": SimpleNamespace(entity=entity), + "household_weight": SimpleNamespace(entity=entity), + } + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable == "household_weight": + return np.ones(N) + return np.full(N, self.balance[int(period)] / N) + + +def test_checks_remember_which_years_failed(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", True) + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken") + record = rec.checks["identity:x"] + assert record["status"] == "fail" and record["failing"] == {"2026": "broken"} + + +def test_a_check_that_did_not_apply_is_skipped_not_passed(): + rec = Recorder() + rec.check("counted_once:x", "x", None, "zero for every unit") + assert rec.checks["counted_once:x"]["status"] == "skipped" + rec.check("counted_once:x", "x", False, "broken") + assert rec.checks["counted_once:x"]["status"] == "fail" + + +def test_a_rebuilt_baseline_must_match_in_every_year(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + drifting = YearSim({2025: 10.0, 2026: 20.5}) + measure_reforms(lambda reform=None: drifting, None, baseline, {}, [], YEARS, rec) + record = rec.checks["determinism:rebuilt_baseline"] + assert record["status"] == "fail" and list(record["failing"]) == ["2026"] + + rec = Recorder() + measure_reforms( + lambda reform=None: YearSim(baseline), None, baseline, {}, [], YEARS, rec + ) + assert rec.checks["determinism:rebuilt_baseline"]["status"] == "pass" + + +def test_a_non_finite_reform_costing_fails_the_finiteness_check(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + reform = parse_reforms({"reforms": [{"name": "Break it", "parameters": {"a": 1}}]}) + + def factory(reform=None): + return YearSim({2025: 11.0, 2026: float("nan")} if reform else baseline) + + measure_reforms( + factory, + None, + baseline, + {}, + reform, + YEARS, + rec, + baseline=_baseline(factory(), YEARS), + ) + assert rec.checks.get("reform_runs:break_it") is None # recorded by measure() + _finite_metrics_check(rec) + record = rec.checks["finite:metrics"] + assert record["status"] == "fail" and "reforms.break_it" in record["details"][0] + + +def test_the_definitions_digest_follows_what_is_measured(): + reforms = load_reforms() + digest = definitions_digest(SPEC, reforms) + assert digest == definitions_digest(load_spec(), load_reforms()) + shorter = dataclasses.replace(SPEC, last_year=SPEC.last_year - 1) + assert definitions_digest(shorter, reforms) != digest + assert definitions_digest(SPEC, reforms[:-1]) != digest + other_measures = dataclasses.replace(SPEC, raw={**SPEC.raw, "poverty": {}}) + assert definitions_digest(other_measures, reforms) != digest + + +@settings(max_examples=50, deadline=None) +@given(first=st.integers(2020, 2030), length=st.integers(1, 8), drop=st.integers(0, 9)) +def test_the_digest_separates_different_horizons_and_reform_sets(first, length, drop): + reforms = load_reforms() + a = dataclasses.replace(SPEC, first_year=first, last_year=first + length - 1) + b = dataclasses.replace(SPEC, first_year=first, last_year=first + length) + assert definitions_digest(a, reforms) != definitions_digest(b, reforms) + fewer = reforms[:drop] + reforms[drop + 1 :] + assert definitions_digest(a, fewer) != definitions_digest(a, reforms) + + +def test_the_long_format_report_carries_each_checks_failing_years(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", False, "broken in 2025") + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken in 2026") + rec.scope = "" + rec.check("closure", "closure", True) + rec.check("counted_once:y", "y", None, "zero for every unit") + rec.metric("receipts.income_tax", "Income tax", "receipts", "gbp", 2025, 1.0) + report = to_report( + { + "status": "ok", + "years": [2025, 2026], + "provenance": {"metrics_version": "v"}, + "partition": {"sha256": "p"}, + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + } + ) + rows = {row["identity_id"]: row for row in report["identities"]} + assert rows["identity:x"]["failing_scopes"] == ["2025", "2026"] + assert rows["identity:x"]["status"] == "fail" + assert rows["closure"]["failing_scopes"] == [] + assert rows["counted_once:y"]["status"] == "skipped" + assert rows["counted_once:y"]["failing_scopes"] == [] + metric = report["metrics"][0] + assert ( + metric["metric_id"] == "receipts.income_tax" + and metric["partition"] == "income_tax" + ) + assert metric["kind"] == "receipt" and metric["year"] == 2025 + + +def test_the_tool_digest_is_taken_at_load_and_matches_the_files(): + assert tool_digest() == _digest_tool_files() + assert len(tool_digest()) == 64 + + +class TargetSim(YearSim): + """gov_balance by year, plus a target variable whose change a reform sets.""" + + def __init__(self, balance_by_year, target_by_year): + super().__init__(balance_by_year) + self.target = target_by_year + self.tax_benefit_system.variables["universal_credit"] = SimpleNamespace( + entity=SimpleNamespace(key="household") + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable == "universal_credit": + return np.full(N, self.target[int(period)] / N) + return super().calculate(variable, period, map_to, unweighted) + + +def test_responsiveness_is_recorded_per_year(): + """A reform that goes inert in a second year is a regression, not old news.""" + reform = parse_reforms( + { + "reforms": [ + { + "name": "Raise UC", + "parameters": {"a": 1}, + "responds": { + "variable": "universal_credit", + "direction": "up", + "minimum": 5.0, + }, + } + ] + } + ) + baseline = {2025: 0.0, 2026: 0.0} + targets = {2025: {"universal_credit": 100.0}, 2026: {"universal_credit": 100.0}} + + def run(reformed_uc): + rec = Recorder() + + def factory(reform=None): + uc = reformed_uc if reform else {2025: 100.0, 2026: 100.0} + return TargetSim(baseline, uc) + + measure_reforms( + factory, + None, + baseline, + targets, + reform, + YEARS, + rec, + baseline=_baseline(factory(), YEARS, ["universal_credit"]), + ) + return rec.checks["responsive:raise_uc"] + + one_year = run({2025: 110.0, 2026: 100.0}) + both_years = run({2025: 100.0, 2026: 100.0}) + assert one_year["status"] == "fail" and set(one_year["failing"]) == {"2026"} + assert set(both_years["failing"]) == {"2025", "2026"} + verdict = classify_check( + "fail", "fail", False, one_year["failing"], both_years["failing"] + ) + assert verdict == "regression" + + +@settings(max_examples=100, deadline=None) +@given(count=st.integers(0, 10_000), amount=st.floats(-1e9, 1e9, allow_nan=False)) +def test_a_small_cell_never_reveals_an_amount(count, amount): + text = cell_detail(count, "households differ", f"by a weighted £{amount:,.0f}") + if count < MIN_CELL: + assert "£" not in text and str(count) not in text.replace(str(MIN_CELL), "") + else: + assert f"{count:,}" in text and "£" in text + + +def _protection_partition(): + from policyengine_uk.utils.aggregate_impact.partition import parse_partition + + def head(i, totals, kind="spending"): + return { + "id": i, + "label": i, + "variable": i, + "totals": totals, + "entity": "household", + } + + return parse_partition( + { + "schema_version": 1, + "totals": {"gov_tax": {}, "gov_spending": {}, "household_benefits": {}}, + "tax_heads": [head("tax", ["gov_tax"])], + "spending_heads": [ + head("common", ["gov_spending", "household_benefits"]), + head("rare", ["gov_spending", "household_benefits"]), + ], + } + ) + + +HW = np.ones(2) +MARKET = np.array([30_000.0, 20_000.0]) + + +def _rebuilt(common, rare, tax=5e11, records=1, scale=1.0, year=2026, market=MARKET): + """A two-household year as measure_year records it, then rebuilt. + + Household 1 has the common benefit and the tax, household 2 the rare + benefit, from ``records`` records. The model's own totals scale their + heads by ``scale``, as a formula that uprates its components would (Sol + re-review 7 of #2335). Returns the recorder and the rebuilt household + net income and what it leaves out. + """ + from policyengine_uk.utils.aggregate_impact.measure import ( + rebuild_protected, + suppressed_components, + ) + + partition = _protection_partition() + component_hh = { + "common": np.array([common, 0.0]), + "rare": np.array([0.0, rare]), + "tax": np.array([tax, 0.0]), + } + rec = Recorder() + rec.metric("receipts.tax", "Tax", "receipts", "gbp", year, tax, records=5000) + rec.metric("spending.common", "C", "spending", "gbp", year, common, records=500) + rec.metric("spending.rare", "R", "spending", "gbp", year, rare, records=records) + spending = scale * (common + rare) + for total in ("model.gov_spending", "model.household_benefits"): + rec.metric(total, total, "model_totals", "gbp", year, spending) + rec.metric("model.gov_tax", "gov_tax", "model_totals", "gbp", year, scale * tax) + rec.metric( + "model.gov_balance", + "b", + "model_totals", + "gbp", + year, + scale * (tax - common - rare), + ) + rec.metric("balance.receipts", "r", "balance", "gbp", year, tax) + rec.metric("balance.spending", "s", "balance", "gbp", year, common + rare) + rec.metric("balance.balance", "b", "balance", "gbp", year, tax - common - rare) + rec.metric( + "household.household_net_income", + "n", + "household_totals", + "gbp", + year, + float(MARKET.sum()) + spending, + ) + income, excludes = rebuild_protected( + rec, + partition, + set(), + suppressed_components(rec, partition, year), + component_hh, + HW, + year, + market_hh=market, + pension_hh=None if market is None else np.zeros(2), + ) + return rec, income, excludes + + +def _published(rec, year=2026): + return {m: r["values"].get(str(year)) for m, r in rec.metrics.items()} + + +def test_a_total_is_rebuilt_without_its_suppressed_head(): + """Sol re-reviews 5 to 7 of #2335: each total holding a suppressed head is + rebuilt from its other heads, never as the model's total less the head.""" + rec, _, _ = _rebuilt(common=7.3e10, rare=246.9) + published = _published(rec) + assert published["spending.rare"] is None + for total in ("model.gov_spending", "model.household_benefits", "balance.spending"): + assert published[total] == 7.3e10 + assert rec.metrics[total]["excludes"] == {"2026": {"spending.rare": 1}} + # A spending head enters the balances negative. + for total in ("balance.balance", "model.gov_balance"): + assert published[total] == 5e11 - 7.3e10 + assert rec.metrics[total]["excludes"] == {"2026": {"spending.rare": -1}} + # Household net income is rebuilt from market income and its parts. + assert published["household.household_net_income"] == 50_000.0 + 7.3e10 + # A total with no suppressed head is the model's, and marked nothing. + assert published["model.gov_tax"] == 5e11 + assert "excludes" not in rec.metrics["model.gov_tax"] + # The raw value is kept off the record. + assert all(246.9 not in r["values"].values() for r in rec.metrics.values()) + + +@pytest.mark.parametrize("scale", [1.0001, 1.1]) +def test_a_total_that_scales_its_heads_gives_nothing_back(scale): + """Sol re-review 7 of #2335, P2 1: with a tiny uplift in benefits, total + less head kept a residual of the head within the identity's £1 + tolerance. The rebuilt total is the published heads' sum, exactly.""" + rec, _, _ = _rebuilt(common=20_000.0, rare=1_000.0, scale=scale) + published = _published(rec) + assert published["model.household_benefits"] == 20_000.0 + assert published["model.gov_spending"] == 20_000.0 + # A total holding no suppressed head is still the model's own. + assert published["model.gov_tax"] == scale * 5e11 + + +@settings(max_examples=200, deadline=None) +@given( + common=st.floats(1e3, 5e11), + rare=st.floats(-1e10, 1e10), + other=st.floats(-1e10, 1e10), + scale=st.floats(0.5, 2.0), +) +def test_no_published_figure_depends_on_a_suppressed_head(common, rare, other, scale): + """For all values: two datasets that differ only in a suppressed head + publish exactly the same figures, whatever its size or sign and whatever + the model's formulas do with it.""" + first, income_1, ex_1 = _rebuilt(common, rare, scale=scale) + second, income_2, ex_2 = _rebuilt(common, other, scale=scale) + assert _published(first) == _published(second) + np.testing.assert_array_equal(income_1, income_2) + assert ex_1 == ex_2 + + +@settings(max_examples=100, deadline=None) +@given( + commons=st.lists(st.floats(1e8, 5e11), min_size=2, max_size=6), + rare=st.floats(0, 4.99e8), +) +def test_repeated_reports_say_nothing_about_a_suppressed_head(commons, rare): + """Each published total is the published heads' sum in every report, so + no sequence of reports narrows the suppressed head down.""" + for common in commons: + published = _published(_rebuilt(common, rare)[0]) + assert published["model.gov_spending"] == published["spending.common"] + assert published["balance.balance"] == ( + published["receipts.tax"] - published["spending.common"] + ) + + +def test_income_for_deciles_is_rebuilt_without_suppressed_heads(): + _, income, excludes = _rebuilt(common=900.0, rare=900.0) + np.testing.assert_array_equal(income, [30_900.0, 20_000.0]) + assert excludes == {"spending.rare": 1} + # With nothing suppressed, the model's own income is used. + _, income, excludes = _rebuilt(common=900.0, rare=900.0, records=500) + assert income is None and excludes == {} + + +def test_household_net_income_is_suppressed_when_it_cant_be_rebuilt(): + rec, income, excludes = _rebuilt(common=900.0, rare=900.0, market=None) + assert income is None and excludes == {"spending.rare": 1} + assert _published(rec)["household.household_net_income"] is None + assert rec.metrics["household.household_net_income"]["suppressed"] == ["2026"] + + +def test_no_head_is_part_of_household_market_income(): + """Market income holds no tax or benefit head, so it needs no exclusion.""" + from policyengine_uk.utils.aggregate_impact.partition import load_partition + from policyengine_uk.variables.household.income.household_market_income import ( + HOUSEHOLD_MARKET_INCOME_VARIABLES, + ) + + from policyengine_uk.utils.aggregate_impact.partition import default_partition_path + + heads = {head.variable for head in load_partition(default_partition_path()).heads} + assert not heads & set(HOUSEHOLD_MARKET_INCOME_VARIABLES) + + +def test_a_zero_weight_record_does_not_count_towards_a_published_figure(): + from policyengine_uk.utils.aggregate_impact.measure import _weighted_records + + values = np.array([5.0, 5.0, 5.0] + [5.0] * 7 + [0.0] * 5) + weight = np.array([100.0] * 3 + [0.0] * 7 + [100.0] * 5) + # Ten records have values, but only three carry weight: a small cell. + assert _weighted_records(values != 0, weight) == 3 + + +def test_identity_diagnostics_count_only_households_with_weight(): + """Sol re-review 4, finding 4: one weighted household and nine with no + weight differing is a small cell, and its £100 is not published.""" + from policyengine_uk.utils.aggregate_impact.measure import _identity_check + from policyengine_uk.utils.aggregate_impact.spec import load_spec + + rec = Recorder() + components = np.zeros(20) + components[:10] = 100.0 + weights = np.zeros(20) + weights[0] = 1.0 + weights[10:] = 1.0 + _identity_check( + rec, "identity:x", "x", components, np.zeros(20), 2026, load_spec(), weights + ) + (detail,) = rec.checks["identity:x"]["failing"].values() + assert "fewer than 10" in detail and "£" not in detail + + +def test_identities_leave_out_households_holding_a_suppressed_figure(): + """Sol re-review 7 of #2335: with a formula that scales its components, + whether a holder's household differs, and by how much, depends on the + suppressed figure. The verdict and detail come from the others only.""" + from policyengine_uk.utils.aggregate_impact.measure import _identity_check + + components = np.full(20, 100.0) + holding = np.zeros(20, dtype=bool) + holding[0] = True + + def detail(total, leave_out): + rec = Recorder() + _identity_check( + rec, + "identity:x", + "x", + components, + total, + 2026, + load_spec(), + np.ones(20), + leave_out=leave_out, + ) + return rec.checks["identity:x"] + + # Only the holder differs: nothing to report about it. + only_holder = components.copy() + only_holder[0] = 5_000.0 + assert detail(only_holder, holding)["status"] == "pass" + assert detail(only_holder, None)["status"] == "fail" + # Every household differs: the holder isn't counted or summed. + scaled = components * 1.1 + (text,) = detail(scaled, holding)["details"] + assert "19 households differ" in text and "£-190" in text + assert "aren't checked this year" in text + + +class StubSim: + """Every variable household-level, and zero unless given.""" + + def __init__(self, values, n=N): + self.values, self.n = values, n + entity = SimpleNamespace(key="household") + + class Variables(dict): + def __missing__(self, key): + return SimpleNamespace(entity=entity) + + names = set(values) | { + "gov_tax", + "gov_spending", + "gov_balance", + "household_tax", + "household_benefits", + "pre_budget_change_household_tax", + "household_market_income", + "household_net_income", + "pension_contributions", + "hbai_household_net_income", + "hbai_household_net_income_before_reset_to_zero", + } + self.tax_benefit_system = SimpleNamespace( + variables=Variables({v: SimpleNamespace(entity=entity) for v in names}), + parameters=SimpleNamespace(), + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable.endswith("_weight"): + return np.ones(self.n) + if variable == "household_id": + return np.arange(self.n) + return np.asarray(self.values.get(variable, np.zeros(self.n)), dtype=float) + + +def _measured_year(values): + from policyengine_uk.utils.aggregate_impact.measure import measure_year + from policyengine_uk.utils.aggregate_impact.partition import ( + default_partition_path, + load_partition, + ) + + rec, households = Recorder(), {} + rec.scope = "2026" + measure_year( + StubSim(values), + load_partition(default_partition_path()), + load_spec(), + 2026, + rec, + households, + ) + return rec, households + + +def _first(n, value): + out = np.zeros(N) + out[:n] = value + return out + + +def test_a_double_count_from_few_records_is_suppressed_and_left_out(): + """Sol re-review 7 of #2335, P2 4: a corporate SDLT double count from one + record was published, and gov_tax less receipts gave it back too. The + model's totals here also scale their parts by 10%, so the identities fail + with a gap that holds it.""" + sdlt = _first(10, 1_000.0) + corporate = _first(1, 1_234.5) + sdlt = sdlt + corporate # expected_sdlt contains corporate_sdlt + model_tax = 1.1 * (sdlt + corporate) + rec, _ = _measured_year( + { + "expected_sdlt": sdlt, + "corporate_sdlt": corporate, + "gov_tax": model_tax, + "household_tax": model_tax, + "pre_budget_change_household_tax": model_tax, + "gov_balance": model_tax, + } + ) + published = _published(rec) + assert published["double_counts.corporate_sdlt"] is None + assert published["model.gov_tax"] == published["balance.receipts"] == sdlt.sum() + assert rec.metrics["model.gov_tax"]["excludes"]["2026"] == { + "double_counts.corporate_sdlt": 1 + } + assert published["model.gov_balance"] == sdlt.sum() + assert rec.checks["identity:gov_tax"]["status"] == "fail" + assert all("£" not in d for d in rec.checks["identity:gov_tax"]["details"]) + + +def test_an_uprated_benefit_total_is_rebuilt_from_its_published_heads(): + """Sol re-review 7 of #2335, P2 1, through measure_year: ten £2,000 + pensions, one suppressed £1,000 DSA award and an uplift of 0.01% in the + model's totals. The published totals are the pensions' sum exactly.""" + pension = _first(10, 2_000.0) + dsa = np.zeros(N) + dsa[10] = 1_000.0 + benefits = 1.0001 * (pension + dsa) + market = np.full(N, 10_000.0) + rec, households = _measured_year( + { + "state_pension": pension, + "disabled_students_allowance": dsa, + "gov_spending": benefits, + "household_benefits": benefits, + "gov_balance": -benefits, + "household_market_income": market, + "household_net_income": market + benefits, + } + ) + published = _published(rec) + assert published["spending.disabled_students_allowance"] is None + for total in ("model.gov_spending", "model.household_benefits"): + assert published[total] == 20_000.0 + assert published["model.gov_balance"] == -20_000.0 + assert published["household.household_net_income"] == market.sum() + 20_000.0 + # The decile incomes leave it out too. + np.testing.assert_array_equal(households["2026"]["income"], market + pension) + assert list(households["2026"]["excludes"]) == [ + "spending.disabled_students_allowance" + ] + + +AWARDS = st.floats(1.0, 1e6) | st.floats(-1e6, -1.0) + + +@settings(max_examples=30, deadline=None) +@given(award=AWARDS, other=AWARDS, uplift=st.floats(0.9, 1.1)) +def test_no_figure_measure_year_publishes_depends_on_a_suppressed_award( + award, other, uplift +): + """For all awards, of either sign, and uplifts: two datasets that differ + only in one suppressed DSA award publish the same figures, decile + incomes, check statuses and check details.""" + + def run(dsa_award): + pension = _first(10, 2_000.0) + dsa = np.zeros(N) + dsa[10] = dsa_award + benefits = uplift * (pension + dsa) + market = np.full(N, 10_000.0) + return _measured_year( + { + "state_pension": pension, + "disabled_students_allowance": dsa, + "gov_spending": benefits, + "household_benefits": benefits, + "gov_balance": -benefits, + "household_market_income": market, + "household_net_income": market + benefits, + } + ) + + (rec_1, hh_1), (rec_2, hh_2) = run(award), run(other) + first, second = _published(rec_1), _published(rec_2) + # The model's own totals that hold the award are never published as such. + # (A rate with no one in its group is NaN on both sides.) + assert first.keys() == second.keys() + for metric_id, value in first.items(): + assert value == second[metric_id] or ( + value is not None and np.isnan(value) and np.isnan(second[metric_id]) + ), metric_id + np.testing.assert_array_equal(hh_1["2026"]["income"], hh_2["2026"]["income"]) + assert [(c["id"], c["status"], c["details"]) for c in rec_1.checks.values()] == [ + (c["id"], c["status"], c["details"]) for c in rec_2.checks.values() + ] + + +class ArraySim: + """Household arrays by variable for one year; gov_balance is income tax + less the state pension unless given.""" + + def __init__(self, values, n=N): + self.values, self.n = values, n + entity = SimpleNamespace(key="household") + names = set(values) | {"gov_balance", "household_weight"} + self.tax_benefit_system = SimpleNamespace( + variables={v: SimpleNamespace(entity=entity) for v in names} + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable.endswith("_weight"): + return np.ones(self.n) + if variable == "gov_balance" and variable not in self.values: + return self.values["income_tax"] - self.values["state_pension"] + return np.asarray(self.values.get(variable, np.zeros(self.n)), dtype=float) + + +def _pension_partition(): + from policyengine_uk.utils.aggregate_impact.partition import parse_partition + + def head(i, totals): + return {"id": i, "label": i, "variable": i, "totals": totals} + + return parse_partition( + { + "schema_version": 1, + "totals": {"gov_tax": {}, "gov_spending": {}}, + "tax_heads": [head("income_tax", ["gov_tax"])], + "spending_heads": [head("state_pension", ["gov_spending"])], + } + ) + + +def _reform_run(base, reformed, target="state_pension", year=2026): + """One reform measured against a baseline year, as measure() runs it.""" + from policyengine_uk.utils.aggregate_impact.measure import ( + _weighted_records, + baseline_records, + weighted_total, + ) + + partition = _pension_partition() + base_sim = ArraySim(base) + rec = Recorder() + for head in partition.heads: + section = "receipts" if head.kind == "tax" else "spending" + rec.metric( + f"{section}.{head.id}", + head.label, + section, + "gbp", + year, + float(base[head.variable].sum()), + records=_weighted_records(base[head.variable] != 0, np.ones(N)), + ) + reform = parse_reforms( + { + "reforms": [ + { + "name": "Change it", + "parameters": {"a": 1}, + "responds": { + "variable": target, + "direction": "up", + "minimum": 1e12, + }, + } + ] + } + ) + measure_reforms( + lambda reform=None: ArraySim(reformed if reform else base), + None, + {year: weighted_total(base_sim, "gov_balance", year)}, + {year: {target: weighted_total(base_sim, target, year)}}, + reform, + [year], + rec, + baseline={year: baseline_records(base_sim, partition, rec, year, [target])}, + components=partition.component_variables(), + ) + costing = rec.metrics["reforms.change_it"]["values"][str(year)] + change = rec.metrics["reform_targets.change_it"]["values"][str(year)] + return rec, costing, change + + +def _world(pensions, tax): + return { + "state_pension": np.asarray(pensions, dtype=float), + "income_tax": np.asarray(tax, dtype=float), + } + + +def test_cancelling_changes_to_suppressed_records_publish_nothing(): + """Sol re-review 7 of #2335, P2 2: two suppressed pensions move from + £1,000 and £2,000 to £1,100 and £1,900, so their total stays £3,000, but + the first recipient's 20% tax gave a £20 costing, and back the pension.""" + tax = _first(12, 100.0) + tax[0] = 200.0 + base = _world(_first(2, 0.0) + np.r_[1_000.0, 2_000.0, np.zeros(N - 2)], tax) + reformed_tax = tax.copy() + reformed_tax[0] = 220.0 + reformed = _world(np.r_[1_100.0, 1_900.0, np.zeros(N - 2)], reformed_tax) + rec, costing, change = _reform_run(base, reformed) + assert costing is None and change is None + detail = rec.checks["responsive:change_it"]["failing"]["2026"] + assert "£" not in detail and "withheld" in detail + + +def test_a_tiny_change_to_a_suppressed_record_publishes_nothing(): + """Sol re-review 7 of #2335, P2 2: a £4 pension uplifted 10% published a + £0.40 target change under the old £0.50 national threshold.""" + base = _world(_first(1, 4.0), _first(12, 100.0)) + reformed = _world(_first(1, 4.4), _first(12, 100.0)) + _, costing, change = _reform_run(base, reformed) + assert costing is None and change is None + + +def test_a_reform_that_leaves_a_small_cell_publishes_nothing(): + """Sol re-review 7 of #2335, P2 3: nine people paying £46 and one £1,486 + of income tax; a £13,000 allowance leaves £1,400 on one record, which + the −£500 change gave back.""" + tax = np.r_[np.full(9, 46.0), 1_486.0, np.zeros(N - 10)] + reformed_tax = np.r_[np.zeros(9), 1_400.0, np.zeros(N - 10)] + pension = np.full(N, 100.0) + _, costing, change = _reform_run( + _world(pension, tax), _world(pension, reformed_tax), target="income_tax" + ) + assert costing is None and change is None + + +@pytest.mark.parametrize("changed, published", [(3, False), (9, False), (10, True)]) +def test_a_reform_costing_from_fewer_than_10_changed_households_is_withheld( + changed, published +): + tax = np.full(N, 100.0) + reformed_tax = tax.copy() + reformed_tax[:changed] += 10.0 + pension = np.full(N, 100.0) + _, costing, change = _reform_run( + _world(pension, tax), _world(pension, reformed_tax), target="income_tax" + ) + if published: + assert costing == pytest.approx(10.0 * changed) + assert change == pytest.approx(10.0 * changed) + else: + assert costing is None and change is None + + +@settings(max_examples=100, deadline=None) +@given( + pensions=st.lists(st.floats(1.0, 1e5), min_size=1, max_size=9), + index=st.integers(0, 8), + change=st.floats(-1e4, 1e4).filter(lambda x: abs(x) > 1e-3), + cancel=st.booleans(), +) +def test_a_reform_that_touches_a_suppressed_record_publishes_nothing( + pensions, index, change, cancel +): + """For all suppressed pensions and any change to one of them, cancelled + nationally by another or not: neither costing nor target change is + published.""" + base = _world(np.r_[pensions, np.zeros(N - len(pensions))], np.full(N, 100.0)) + moved = base["state_pension"].copy() + i = index % len(pensions) + moved[i] += change + if cancel and len(pensions) > 1: + moved[(i + 1) % len(pensions)] -= change + _, costing, change_published = _reform_run(base, _world(moved, base["income_tax"])) + assert costing is None and change_published is None + + +def test_a_reform_without_baseline_records_publishes_nothing(): + rec = Recorder() + baseline = {2025: 0.0, 2026: 0.0} + reform = parse_reforms({"reforms": [{"name": "Any", "parameters": {"a": 1}}]}) + measure_reforms( + lambda reform=None: YearSim({2025: 5.0, 2026: 5.0} if reform else baseline), + None, + baseline, + {}, + reform, + YEARS, + rec, + ) + assert rec.metrics["reforms.any"]["values"] == {"2025": None, "2026": None} + + +def test_hbai_holds_a_component_only_when_it_is_one_of_its_parts(): + from policyengine_uk.utils.aggregate_impact.measure import hbai_holds_any + + assert not hbai_holds_any({"disabled_students_allowance": ("x", 1.0)}) + assert hbai_holds_any({"child_benefit": ("x", 1.0)}) + + +class HbaiSim: + """Household-level HBAI pieces: employment income, child benefit and + council tax, and the model's HBAI income before its reset.""" + + def __init__(self, before, council_tax=1_200.0): + self.values = { + "employment_income": np.array([20_000.0, 0.0]), + "child_benefit": np.array([1_234.5, 100.0]), + "council_tax": np.array([council_tax, 0.0]), + "hbai_household_net_income_before_reset_to_zero": before, + "hbai_household_net_income": np.maximum(0.0, before), + } + self.tax_benefit_system = SimpleNamespace(variables=self.values) + + def calculate(self, variable, period, map_to=None, unweighted=True): + return self.values.get(variable, np.zeros(2)) + + +HIDDEN_CB = {"child_benefit": ("spending.child_benefit", 1_334.5)} + + +def test_hbai_is_rebuilt_without_the_suppressed_component(): + from policyengine_uk.utils.aggregate_impact.measure import _hbai_without + + consistent = np.array([20_000.0 + 1_234.5 - 1_200.0, 100.0]) + values, excludes = _hbai_without(HbaiSim(consistent), 2026, HIDDEN_CB) + np.testing.assert_array_equal(values, [18_800.0, 0.0]) + assert excludes == {"spending.child_benefit": 1} + + +@pytest.mark.parametrize("council_tax", [1_200.0, 0.5, 0.01]) +def test_hbai_is_suppressed_where_its_lists_dont_reproduce_the_model(council_tax): + """Sol re-reviews 6 and 7 of #2335: with council tax abolished the model + leaves it out of HBAI. Even £0.50 of it passed the old £1 tolerance and + came back in.""" + from policyengine_uk.utils.aggregate_impact.measure import _hbai_without + + abolished = np.array([20_000.0 + 1_234.5, 100.0]) + assert _hbai_without(HbaiSim(abolished, council_tax), 2026, HIDDEN_CB) == ( + None, + None, + ) + + +def test_hbai_reproduction_is_not_judged_on_households_holding_the_figure(): + from policyengine_uk.utils.aggregate_impact.measure import _hbai_without + + # The model leaves council tax out for the first household only, which + # holds the suppressed child benefit: its verdict would depend on it. + before = np.array([20_000.0 + 1_234.5, 100.0 - 0.0]) + holding = np.array([True, False]) + values, excludes = _hbai_without(HbaiSim(before), 2026, HIDDEN_CB, holding) + np.testing.assert_array_equal(values, [18_800.0, 0.0]) + assert _hbai_without(HbaiSim(before), 2026, HIDDEN_CB) == (None, None) + + +def test_a_suppressed_hbai_component_withholds_deciles_poverty_and_the_gini(): + """Households are ranked into deciles on HBAI income: with a suppressed + child benefit award in it, which decile its holder lands in depends on + the award.""" + child_benefit = np.zeros(N) + child_benefit[0] = 1_234.5 + market = np.full(N, 10_000.0) + rec, households = _measured_year( + { + "child_benefit": child_benefit, + "gov_spending": child_benefit, + "household_benefits": child_benefit, + "gov_balance": -child_benefit, + "household_market_income": market, + "household_net_income": market + child_benefit, + } + ) + assert "2026" not in households + for record in rec.metrics.values(): + if record["section"] in ("deciles", "poverty", "poverty_count", "inequality"): + assert record["values"]["2026"] is None, record["id"] + assert "2026" in record["suppressed"] + + +@settings(max_examples=100, deadline=None) +@given(award=st.floats(0, 1e5), other=st.floats(0, 1e5)) +def test_rebuilt_hbai_never_depends_on_the_suppressed_component(award, other): + from policyengine_uk.utils.aggregate_impact.measure import _hbai_without + + def rebuilt(cb): + sim = HbaiSim(np.array([20_000.0 + cb - 1_200.0, 100.0])) + sim.values["child_benefit"] = np.array([cb, 100.0]) + return _hbai_without(sim, 2026, HIDDEN_CB)[0] + + np.testing.assert_array_equal(rebuilt(award), rebuilt(other)) + + +def test_a_suppressed_heads_sign_and_finite_checks_say_nothing_about_it(): + """Sol re-review 8 of #2335, P3 2: whether one of so few records is + negative is about it alone, and a count of non-finite values under 10 is + a small cell.""" + dsa = np.zeros(N) + dsa[10] = -1_000.0 + rec, _ = _measured_year({"disabled_students_allowance": dsa}) + sign = rec.checks["sign:disabled_students_allowance"] + assert sign["status"] == "skipped" + assert "fewer than 10 records" in sign["details"][0] + dsa[10] = np.nan + rec, _ = _measured_year({"disabled_students_allowance": dsa}) + (detail,) = rec.checks["finite:disabled_students_allowance"]["details"] + assert "fewer than 10 non-finite values" in detail + + +@pytest.mark.parametrize("corporate", [1_234.5, 1_000.0]) +def test_the_overlap_guard_leaves_out_households_holding_a_suppressed_figure( + corporate, +): + """Sol re-review 8 of #2335, P3 3: a one-record corporate SDLT in a + household also paying LBTT moved the count from 9 to 10 households.""" + sdlt = _first(10, 1_000.0) + lbtt = _first(10, 500.0) + holder_corporate = np.zeros(N) + holder_corporate[0] = corporate + # The holder's SDLT is all corporate when corporate is 1,000, so its own + # SDLT nets to zero and it would not overlap. + sdlt[0] = max(corporate, 1_000.0) + rec, _ = _measured_year( + { + "expected_sdlt": sdlt, + "expected_lbtt": lbtt, + "corporate_sdlt": holder_corporate, + } + ) + check = rec.checks["overlap_guard:property_transaction_tax_geography"] + (detail,) = check["details"] + assert "fewer than 10 households have more than one" in detail + assert "aren't checked this year" in detail + + +def test_the_published_guard_strips_a_few_unit_double_count_coefficient(): + """Sol re-review 8 of #2335, P3 4.""" + from policyengine_uk.utils.aggregate_impact.measure import publishable_guard + + def guard(units): + return { + "components": [ + {"variable": "corporate_sdlt", "units": units, "status": "pass"} + ], + "double_counts": [ + { + "variable": "corporate_sdlt", + "contained_in": "expected_sdlt", + "coefficient": 0.4321, + "status": "fail", + "detail": "expected_sdlt moves by 0.4321 per unit of corporate_sdlt", + } + ], + } + + (few,) = publishable_guard(guard(3))["double_counts"] + assert few["coefficient"] is None and "0.4321" not in few["detail"] + assert "fewer than 10 units" in few["detail"] + (many,) = publishable_guard(guard(500))["double_counts"] + assert many["coefficient"] == 0.4321 and "0.4321" in many["detail"] + # Sol delta review 9 of #2335, P2: the comment must render it. + from policyengine_uk.utils.aggregate_impact.render import _guard + + for units, shown in ( + (3, "coefficient not shown, fewer than 10 units"), + (500, "coefficient 0.432"), + ): + published = { + "status": "ran", + "year": 2026, + "components": [], + **publishable_guard(guard(units)), + } + published["double_counts"][0]["issue"] = "#2159" + text = _guard({"head": {"guard": published}}) + assert shown in text and "NO LONGER contained" in text + if units == 3: + assert "0.43" not in text diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py new file mode 100644 index 0000000000..f8a8b7dbe9 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py @@ -0,0 +1,221 @@ +"""The fiscal partition against the model's own component lists. + +These run without data. The partition was written from the receipts +classification and each variable's formula; closure compares it with +GOV_TAX_VARIABLES and its siblings, which are a second, independent statement +of the same totals. +""" + +import copy + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.partition import ( + PartitionError, + closure_problems, + default_partition_path, + group_label, + load_partition, + parse_partition, + read_model_lists, + structural_problems, +) + +PATH = default_partition_path() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +@pytest.fixture(scope="module") +def shipped(): + return load_partition(PATH) + + +@pytest.fixture(scope="module") +def raw(): + with open(PATH) as f: + return yaml.safe_load(f) + + +@pytest.fixture(scope="module") +def model_lists(shipped): + lists, problems = read_model_lists(shipped) + assert not problems + return lists + + +def test_shipped_partition_is_found_and_loads(shipped): + assert PATH is not None and PATH.name == "fiscal_partition.yaml" + assert shipped.schema_version == 1 + assert shipped.tax_heads and shipped.spending_heads + + +def test_every_model_total_is_exactly_its_heads_and_declared_double_counts( + shipped, model_lists, system +): + assert set(model_lists) == { + "gov_tax", + "household_tax", + "pre_budget_change_household_tax", + "gov_spending", + "household_benefits", + } + assert closure_problems(shipped, model_lists, system.variables) == [] + + +def test_every_head_is_a_money_variable_with_its_declared_entity(shipped, system): + for head in shipped.heads: + variable = system.variables[head.variable] + assert variable.value_type is float, head.id + assert variable.definition_period in ("year", "month"), head.id + assert head.entity == variable.entity.key, head.id + + +def test_receipts_and_spending_are_separate(shipped): + for head in shipped.tax_heads: + assert not set(head.totals) & {"gov_spending", "household_benefits"} + for head in shipped.spending_heads: + assert not set(head.totals) & {"gov_tax", "household_tax"} + + +def test_dropped_when_matches_the_council_tax_abolition_formulas(shipped): + switch = "gov.contrib.abolish_council_tax" + assert shipped.dropped("gov_tax", [switch]) == { + "council_tax", + "high_value_council_tax_surcharge", + } + assert shipped.dropped("gov_spending", [switch]) == {"council_tax_benefit"} + assert shipped.dropped("gov_tax") == set() + assert set(shipped.components("gov_tax")) - set( + shipped.components("gov_tax", [switch]) + ) == {"council_tax", "high_value_council_tax_surcharge"} + + +def test_group_labels_read_as_sentence_case(): + assert group_label("excise_duties") == "Excise duties" + assert group_label("Income tax and NICs") == "Income tax and NICs" + + +def _remove_head(raw, head_id): + """The partition without one head and every reference to it.""" + out = copy.deepcopy(raw) + removed = next( + h for h in raw["tax_heads"] + raw["spending_heads"] if h["id"] == head_id + ) + for key in ("tax_heads", "spending_heads"): + out[key] = [h for h in out[key] if h["id"] != head_id] + for head in out[key]: + if head.get("reads"): + head["reads"] = [r for r in head["reads"] if r.get("head") != head_id] + out["exclusions"] = [ + e + for e in out.get("exclusions") or [] + if e.get("head") != head_id and head_id not in (e.get("spans") or []) + ] + out["model_double_counts"] = [ + d + for d in out.get("model_double_counts") or [] + if d["contained_in"] != removed["variable"] + ] + for guard in out.get("overlap_guards") or []: + if isinstance(guard, dict) and guard.get("heads"): + guard["heads"] = [h for h in guard["heads"] if h != head_id] + if ( + guard.get("kind") == "geographic_exclusivity" + and len(guard["heads"]) < 2 + ): + guard["kind"] = "removed_with_head" + for spec in (out.get("totals") or {}).values(): + for parameter, variables in ((spec or {}).get("dropped_when") or {}).items(): + spec["dropped_when"][parameter] = [ + v for v in variables if v != removed["variable"] + ] + return out + + +def _mutations(raw): + """Each yields a partition dict that a check must reject, and which check.""" + out = [] + a = copy.deepcopy(raw) + a["tax_heads"][1]["variable"] = a["tax_heads"][0]["variable"] + out.append(("duplicate head variable", a, "structural")) + out.append( + ("missing head", _remove_head(raw, raw["tax_heads"][0]["id"]), "closure") + ) + c = copy.deepcopy(raw) + c["spending_heads"].append( + { + "id": "not_a_model_component", + "label": "x", + "variable": "employment_income", + "totals": ["gov_spending"], + } + ) + out.append(("extra component", c, "closure")) + d = copy.deepcopy(raw) + if d.get("model_double_counts"): + d["model_double_counts"][0]["contained_in"] = "vat" + out.append(("double count in a total its container is not in", d, "structural")) + e = copy.deepcopy(raw) + e["tax_heads"][0]["totals"] = list(e["tax_heads"][0]["totals"]) + ["gov_spending"] + out.append(("tax head in a spending total", e, "structural")) + f = copy.deepcopy(raw) + f["tax_heads"][0]["id"] = f["tax_heads"][1]["id"] + out.append(("duplicate head id", f, "structural")) + return out + + +def test_each_kind_of_broken_partition_is_rejected(raw, model_lists, system): + for name, mutated, check in _mutations(raw): + if check == "structural": + with pytest.raises(PartitionError): + parse_partition(mutated) + else: + partition = parse_partition(mutated) + assert closure_problems(partition, model_lists, system.variables), name + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_component_sets_do_not_depend_on_head_order(raw, data): + shuffled = copy.deepcopy(raw) + shuffled["tax_heads"] = data.draw(st.permutations(shuffled["tax_heads"])) + shuffled["spending_heads"] = data.draw(st.permutations(shuffled["spending_heads"])) + a, b = parse_partition(raw), parse_partition(shuffled) + for total in a.totals: + assert sorted(a.components(total)) == sorted(b.components(total)) + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_removing_any_head_breaks_closure_of_each_total_it_was_in( + raw, model_lists, data +): + heads = raw["tax_heads"] + raw["spending_heads"] + removed = heads[data.draw(st.integers(0, len(heads) - 1))] + problems = closure_problems( + parse_partition(_remove_head(raw, removed["id"])), model_lists + ) + for total in removed["totals"]: + assert any(total in p and removed["variable"] in p for p in problems) + + +def test_structural_checks_pass_on_the_shipped_file(shipped): + assert structural_problems(shipped) == [] + + +def test_a_geographic_guard_must_name_two_known_heads(raw): + for heads in ([], ["council_tax"], ["council_tax", "no_such_head"]): + mutated = copy.deepcopy(raw) + mutated["overlap_guards"] = [ + {"id": "g", "kind": "geographic_exclusivity", "heads": heads} + ] + with pytest.raises(PartitionError): + parse_partition(mutated) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py new file mode 100644 index 0000000000..c6bbd7cddf --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py @@ -0,0 +1,287 @@ +"""The standard reform set and the responsiveness rule.""" + +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.reforms import ( + CALENDAR_DATED_METADATA, + DEFAULT_REFORMS_PATH, + ReformSpecError, + Responds, + calibrate, + calibrated_minimum, + load_reforms, + parameter_node, + parse_reforms, + reform_parameters, + responsiveness, +) +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +TESTS = Path(__file__).resolve().parents[1] +YEARS = load_spec().years +STANDARD = load_reforms() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +def test_every_standard_reform_says_what_it_must_move(system): + assert len(STANDARD) == 10 + assert len({r.id for r in STANDARD}) == len(STANDARD) + for reform in STANDARD: + assert reform.responds is not None, reform.id + assert reform.responds.variable in system.variables, reform.id + assert reform.responds.minimum > 0 + assert bool(reform.parameters) != bool(reform.scale), reform.id + + +def test_standard_set_contains_the_reform_cost_fixtures_unchanged(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f)["reforms"] + by_name = {r.name: r for r in STANDARD} + for fixture in fixtures: + assert dict(by_name[fixture["name"]].parameters) == fixture["parameters"] + + +def test_every_named_parameter_exists(system): + for reform in STANDARD: + for path in list(reform.parameters) + list(reform.scale): + assert parameter_node(system.parameters, path) is not None, path + + +def test_bracket_paths_resolve(system): + node = parameter_node(system.parameters, "gov.hmrc.income_tax.rates.uk[0].rate") + assert node("2026-01-01") == pytest.approx(0.2) + + +def test_scaled_reforms_are_the_baseline_times_the_factor_in_every_year(system): + for reform in STANDARD: + changes = reform_parameters(reform, system.parameters, YEARS) + for path, factor in reform.scale.items(): + assert set(changes[path]) == {str(y) for y in YEARS} + node = parameter_node(system.parameters, path) + for year in YEARS: + assert changes[path][str(year)] == pytest.approx( + node(f"{year}-01-01") * factor + ) + for path, value in reform.parameters.items(): + assert changes[path] == value + + +@pytest.mark.parametrize( + "entry", + [ + {"name": "no change"}, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "sideways", "minimum": 1}, + }, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "up", "minimum": 0}, + }, + {"name": "x", "scale": {"a": 0}}, + ], +) +def test_malformed_reforms_are_rejected(entry): + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry]}) + + +def test_duplicate_ids_are_rejected(): + entry = {"name": "Same name", "parameters": {"a": 1}} + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry, dict(entry)]}) + + +def test_fixture_schema_parses_without_responds(): + reforms = parse_reforms({"reforms": [{"name": "Raise X", "parameters": {"a": 1}}]}) + assert reforms[0].id == "raise_x" and reforms[0].responds is None + + +changes = st.dictionaries( + st.integers(2025, 2035), + st.floats(-1e11, 1e11, allow_nan=False), + min_size=1, + max_size=8, +) + + +@settings(max_examples=200, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10)) +def test_responsive_exactly_when_every_year_moves_enough_the_right_way( + changes, minimum +): + up, _ = responsiveness(Responds("v", "up", minimum), changes) + assert up == all(c >= minimum for c in changes.values()) + # Mirror image: the same reform read as a fall of the negated changes. + down, _ = responsiveness( + Responds("v", "down", minimum), {y: -c for y, c in changes.items()} + ) + assert down == up + + +@settings(max_examples=100, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10), year=st.integers(2025, 2035)) +def test_one_inert_or_wrong_signed_year_fails(changes, minimum, year): + for bad in (0.0, -abs(minimum)): + ok, detail = responsiveness( + Responds("v", "up", minimum), {**changes, year: bad} + ) + assert not ok and str(year)[2:] in detail + + +def test_no_measured_year_fails(): + ok, detail = responsiveness(Responds("v", "up", 1.0), {}) + assert not ok and "no year" in detail + + +def test_a_calendar_dated_parameter_cannot_be_scaled(system): + dated = [] + + def walk(node, path): + for name, child in getattr(node, "children", {}).items(): + child_path = f"{path}.{name}" if path else name + metadata = getattr(child, "metadata", None) or {} + if any(metadata.get(k) for k in CALENDAR_DATED_METADATA) and hasattr( + child, "values_list" + ): + dated.append(child_path) + walk(child, child_path) + + walk(system.parameters, "") + assert dated, "expected at least one calendar-dated parameter in the model" + reform = parse_reforms( + {"reforms": [{"name": "Scale it", "scale": {dated[0]: 1.1}}]} + )[0] + with pytest.raises(ReformSpecError): + reform_parameters(reform, system.parameters, YEARS) + + +def test_a_scaled_reform_sets_exactly_the_horizon_years_in_the_model(): + """Build the reformed system the way a simulation does and read it back.""" + from policyengine_uk import Simulation + + reform = next(r for r in STANDARD if r.id == "uc_standard_allowance_plus_10pc") + situation = {"people": {"a": {"age": 30}}} + baseline = Simulation(situation=situation) + changes = reform_parameters(reform, baseline.tax_benefit_system.parameters, YEARS) + reformed = Simulation(situation=situation, reform=changes) + for path, factor in reform.scale.items(): + before = parameter_node(baseline.tax_benefit_system.parameters, path) + after = parameter_node(reformed.tax_benefit_system.parameters, path) + for year in YEARS: + assert after(f"{year}-06-01") == pytest.approx( + before(f"{year}-06-01") * factor + ) + outside = YEARS[-1] + 1 + assert after(f"{outside}-06-01") == pytest.approx(before(f"{outside}-06-01")) + assert after(f"{YEARS[0] - 1}-06-01") == pytest.approx( + before(f"{YEARS[0] - 1}-06-01") + ) + + +def test_the_shipped_minimums_are_a_fifth_of_each_smallest_effect(): + """The d1344 rule, as parse_reforms enforces it on the shipped file.""" + raw = yaml.safe_load(DEFAULT_REFORMS_PATH.read_text()) + assert raw["calibration"]["fraction"] == 0.2 + for reform in STANDARD: + r = reform.responds + assert r.smallest_effect is not None, reform.id + assert r.minimum == calibrated_minimum(r.smallest_effect, 0.2), reform.id + assert 0.18 * r.smallest_effect < r.minimum <= 0.2 * r.smallest_effect + + +@settings(max_examples=300, deadline=None) +@given(effect=st.floats(1e3, 1e13), fraction=st.floats(0.01, 0.99)) +def test_a_calibrated_minimum_is_the_fraction_rounded_down_to_two_figures( + effect, fraction +): + minimum = calibrated_minimum(effect, fraction) + amount = effect * fraction + assert 0.9 * amount < minimum <= amount + digits = f"{minimum:.15e}".split("e")[0].replace(".", "").rstrip("0") + assert len(digits) <= 2 + # Monotone: a larger effect never gets a smaller minimum. + assert calibrated_minimum(effect * 1.5, fraction) >= minimum + + +def _calibrated(minimum, smallest=1_000_000_000): + return { + "calibration": {"fraction": 0.2}, + "reforms": [ + { + "name": "x", + "parameters": {"a": 1}, + "responds": { + "variable": "v", + "direction": "up", + "minimum": minimum, + "smallest_effect": smallest, + }, + } + ], + } + + +def test_a_calibrated_file_refuses_a_minimum_edited_by_hand(): + assert parse_reforms(_calibrated(200_000_000))[0].responds.minimum == 2e8 + with pytest.raises(ReformSpecError, match="two significant figures"): + parse_reforms(_calibrated(250_000_000)) + with pytest.raises(ReformSpecError, match="smallest_effect"): + raw = _calibrated(200_000_000) + del raw["reforms"][0]["responds"]["smallest_effect"] + parse_reforms(raw) + with pytest.raises(ReformSpecError, match="fraction"): + raw = _calibrated(200_000_000) + raw["calibration"]["fraction"] = 1.5 + parse_reforms(raw) + + +def _result(changes): + return { + "metrics": [ + {"id": f"reform_targets.{rid}", "values": {str(y): v for y, v in c.items()}} + for rid, c in changes.items() + ] + } + + +def test_calibrate_takes_the_smallest_effect_across_years_and_measurements(): + reforms = parse_reforms( + { + "reforms": [ + { + "id": "up", + "name": "u", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "up", "minimum": 1}, + }, + { + "id": "down", + "name": "d", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "down", "minimum": 1}, + }, + ] + } + ) + results = [ + _result({"up": {2025: 5e9, 2026: 4e9}, "down": {2025: -3e9, 2026: -2.5e9}}), + _result({"up": {2025: 4.5e9, 2026: 3.9e9}, "down": {2025: -3e9, 2026: -2e9}}), + ] + out = calibrate(reforms, results, 0.2) + assert out["up"] == {"smallest_effect": 3.9e9, "year": 2026, "minimum": 7.8e8} + assert out["down"] == {"smallest_effect": 2e9, "year": 2026, "minimum": 4e8} + with pytest.raises(ReformSpecError, match="wrong way"): + calibrate(reforms, [_result({"up": {2025: -1.0}, "down": {2025: -1.0}})], 0.2) diff --git a/policyengine_uk/tests/certified_dataset.yaml b/policyengine_uk/tests/certified_dataset.yaml new file mode 100644 index 0000000000..1ef1809e54 --- /dev/null +++ b/policyengine_uk/tests/certified_dataset.yaml @@ -0,0 +1,20 @@ +# The dataset that credentialed test jobs and the aggregate-impact report run +# on. It is pinned by Hugging Face commit, not by tag, and every run checks the +# file's SHA-256 against `sha256`, so the selection cannot move underneath a +# test. Change it only to another certified pair, together with the reform +# regression fixtures in microsimulation/reforms_config.yaml, which record the +# same dataset. +data_package: policyengine-uk-data +data_version: 1.58.1 +repo_id: policyengine/policyengine-uk-data-private +path: enhanced_frs_2024_25.h5 +# Commit the 1.58.1 tag pointed to on 2026-10-15. +revision: 2526edbbb687a3b54dbb11272e5389e4b62d2ac2 +hf_tag: 1.58.1 +sha256: 056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784 +size_bytes: 128409762 +certification: + certified_by: policyengine.py bundle uk-6.4.3 + certified_for_model_version: 2.130.1 + built_with_model_version: 2.122.2 + release_manifest_revision: 89be6b73fc1e6eeba6dae2f9400676d8510bcd51 diff --git a/policyengine_uk/tests/code_health/test_aggregate_impact_workflow.py b/policyengine_uk/tests/code_health/test_aggregate_impact_workflow.py new file mode 100644 index 0000000000..b9554b3e52 --- /dev/null +++ b/policyengine_uk/tests/code_health/test_aggregate_impact_workflow.py @@ -0,0 +1,97 @@ +"""Data protection: only aggregates may leave the aggregate-impact job. + +The certified dataset is licensed survey microdata. The measurement keeps +household-level detail for the decile comparison under ``private/``; these +tests fail if the workflow could publish it, or could put the dataset where +other workflows can read it. +""" + +import json +from pathlib import Path + +import numpy as np +import pytest +import yaml + +from policyengine_uk.utils.aggregate_impact.measure import ( + HOUSEHOLDS_FILE, + PRIVATE_DIRECTORY, + read_result, + write_result, +) + +REPO = Path(__file__).resolve().parents[3] +WORKFLOW = REPO / ".github" / "workflows" / "aggregate-impact.yaml" + +if not WORKFLOW.exists(): + pytest.skip( + "the .github folder is not part of an installed package", + allow_module_level=True, + ) +ALLOWED_UPLOAD_PATHS = {"report"} + + +def steps(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + return [step for job in workflow["jobs"].values() for step in job["steps"]] + + +def test_only_the_aggregate_report_is_uploaded(): + uploads = [s for s in steps() if "upload-artifact" in str(s.get("uses", ""))] + assert uploads, "the report should be uploaded" + for step in uploads: + paths = {p.strip() for p in str(step["with"]["path"]).splitlines() if p.strip()} + assert paths <= ALLOWED_UPLOAD_PATHS, paths + + +def test_the_dataset_is_never_cached(): + for step in steps(): + assert "actions/cache" not in str(step.get("uses", "")), step.get("name") + + +def test_nothing_private_is_copied_into_the_report(): + text = WORKFLOW.read_text() + copies = [ + line.strip() for line in text.splitlines() if line.strip().startswith("cp ") + ] + for line in copies: + if "report/" in line: + assert line.split()[1].endswith("result.json"), line + assert f"{PRIVATE_DIRECTORY}/" not in "\n".join(copies) + assert HOUSEHOLDS_FILE not in "\n".join(copies) + + +def test_the_job_runs_both_sides_itself(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + assert list(workflow["jobs"]) == ["report"], ( + "base and head must be measured in one job, so that household detail " + "never has to travel between jobs as an artifact" + ) + assert not [s for s in steps() if "download-artifact" in str(s.get("uses", ""))] + + +def test_household_detail_is_written_only_under_private(tmp_path): + households = { + "2026": { + "household_id": np.arange(5), + "household_weight": np.ones(5), + "household_count_people": np.ones(5), + "decile": np.arange(5), + "income": np.linspace(1, 2, 5), + } + } + write_result(tmp_path, {"status": "ok", "metrics": []}, households) + top_level = sorted(p.name for p in tmp_path.iterdir()) + assert top_level == [PRIVATE_DIRECTORY, "result.json"] + assert (tmp_path / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE).exists() + result, loaded = read_result(tmp_path) + assert json.loads((tmp_path / "result.json").read_text()) == result + assert np.array_equal(loaded["2026"]["income"], households["2026"]["income"]) + + +def test_a_failed_run_always_replaces_an_earlier_comment(): + """Even a run that failed before the comparison posts a failing comment.""" + post = next(s for s in steps() if s.get("name", "").startswith("Post or update")) + assert post["if"] == "always() && github.event_name == 'pull_request'" + assert "if [ ! -s report/comment.md ]" in post["run"] + assert "FAIL" in post["run"] diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index e0b262a08c..f306330c83 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -2,23 +2,177 @@ import ctypes import gc import hashlib +import importlib.util import os import re import sys import weakref +from dataclasses import dataclass from importlib.machinery import SourceFileLoader +from pathlib import Path +from typing import Mapping, Optional import pytest from policyengine_core.taxbenefitsystems import TaxBenefitSystem -DEFAULT_TEST_DATASET_URL = ( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.3" +from policyengine_uk.utils.aggregate_impact.dataset import load_pin, materialise + +# Microsimulation tests run on one certified dataset, pinned by commit and +# SHA-256 in certified_dataset.yaml. Found through the package, not beside this +# file: CI's own tests run copies of this conftest from other directories. +CERTIFIED_DATASET_PIN = ( + Path(importlib.util.find_spec("policyengine_uk").origin).parent + / "tests" + / "certified_dataset.yaml" ) +CERTIFIED_DATASET = load_pin(CERTIFIED_DATASET_PIN) +DEFAULT_TEST_DATASET_URL = CERTIFIED_DATASET.uri + +DATASET_ENV_VAR = "POLICYENGINE_UK_DEFAULT_DATASET" +TOKEN_ENV_VAR = "HUGGING_FACE_TOKEN" +# Mandatory-data mode, for CI jobs that must run the microsimulation tests: +# with it set, a test that needs data fails when the certified dataset is +# unavailable, instead of being skipped, and runs only on that dataset. +REQUIRE_DATA_ENV_VAR = "POLICYENGINE_UK_REQUIRE_TEST_DATA" +_TRUE = {"1", "true", "yes", "on"} +_FALSE = {"", "0", "false", "no", "off"} + + +@dataclass(frozen=True) +class TestDataPlan: + """What the microsimulation tests run on, decided from the environment.""" + + __test__ = False # not a test class + + action: str # "run", "skip" or "fail" + dataset: Optional[str] + pinned: bool + mandatory: bool + reason: str + + +def plan_test_data(env: Mapping[str, str], pinned_uri: str) -> TestDataPlan: + """Decide how microsimulation tests get their data. + + - An explicit ``POLICYENGINE_UK_DEFAULT_DATASET`` wins; in mandatory mode + it must be the pinned certified dataset. + - Otherwise a Hugging Face token selects the pinned certified dataset. + - Otherwise the tests skip, or in mandatory mode fail. + + An unrecognised mandatory-mode value fails closed. + """ + flag = str(env.get(REQUIRE_DATA_ENV_VAR, "")).strip().lower() + explicit = env.get(DATASET_ENV_VAR) or None + token = env.get(TOKEN_ENV_VAR) or None + if flag not in _TRUE | _FALSE: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"{REQUIRE_DATA_ENV_VAR}={env.get(REQUIRE_DATA_ENV_VAR)!r} is not a " + "recognised true or false value", + ) + mandatory = flag in _TRUE + if explicit: + pinned = explicit == pinned_uri + if mandatory and not pinned: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"mandatory-data mode runs only on the pinned certified dataset " + f"{pinned_uri}, not {explicit}", + ) + return TestDataPlan( + "run", + explicit, + pinned, + mandatory, + "pinned certified dataset" if pinned else "dataset set explicitly", + ) + if token: + return TestDataPlan( + "run", pinned_uri, True, mandatory, "pinned certified dataset" + ) + if mandatory: + return TestDataPlan( + "fail", + None, + False, + True, + f"mandatory-data mode needs {TOKEN_ENV_VAR} (or {DATASET_ENV_VAR} set " + "to the pinned certified dataset)", + ) + return TestDataPlan( + "skip", + None, + False, + False, + f"Requires {DATASET_ENV_VAR} or {TOKEN_ENV_VAR} for microsimulation " + "dataset access", + ) + + +TEST_DATA_PLAN = plan_test_data(os.environ, DEFAULT_TEST_DATASET_URL) + +if TEST_DATA_PLAN.action == "run" and not os.environ.get(DATASET_ENV_VAR): + os.environ[DATASET_ENV_VAR] = TEST_DATA_PLAN.dataset + + +_verification: dict = {} + -if os.environ.get("HUGGING_FACE_TOKEN") and not os.environ.get( - "POLICYENGINE_UK_DEFAULT_DATASET" -): - os.environ["POLICYENGINE_UK_DEFAULT_DATASET"] = DEFAULT_TEST_DATASET_URL +def verify_certified_dataset() -> Optional[str]: + """Fetch the pinned dataset once per process and check its SHA-256. + + Returns None when the file matches, or why it could not be verified. + """ + if "problem" not in _verification: + try: + materialise(CERTIFIED_DATASET) + _verification["problem"] = None + except Exception as error: # noqa: BLE001 - every cause fails the tests + _verification["problem"] = ( + f"The pinned certified dataset {CERTIFIED_DATASET.describe()} could " + f"not be fetched and verified: {type(error).__name__}: {error}" + ) + return _verification["problem"] + + +# Test files that read the unpinned latest dataset on purpose, to notice when +# a new data release breaks the model. They are data tests (they need +# credentials, and fail without them in mandatory mode), but what they run on +# is by design not the pinned certified dataset. +UNPINNED_BY_DESIGN = ("test_latest_data_smoke.py",) + + +def needs_data(item) -> bool: + """Whether a test carries the microsimulation marker. + + The marker, not the keyword: keywords include directory names, which + would sweep in the data-free tests under ``microsimulation/``. + """ + return item.get_closest_marker("microsimulation") is not None + + +def runs_on_unpinned_data(item) -> bool: + return item.path.name in UNPINNED_BY_DESIGN + + +def pytest_report_header(config): + plan = TEST_DATA_PLAN + mode = "mandatory" if plan.mandatory else "optional" + target = CERTIFIED_DATASET.describe() if plan.pinned else (plan.dataset or "none") + lines = [ + f"microsimulation data ({mode}): {plan.action} on {target} ({plan.reason})", + "unpinned by design: " + ", ".join(UNPINNED_BY_DESIGN), + ] + shard = config.getoption("shard", default=None) + if shard is not None: + lines.append(f"shard: {shard[0]} of {shard[1]}, chosen by node id") + return lines @pytest.fixture(scope="module") @@ -70,13 +224,8 @@ def pytest_addoption(parser): ) -def pytest_report_header(config): - shard = config.getoption("shard", default=None) - if shard is not None: - return f"shard: {shard[0]} of {shard[1]}, chosen by node id" - - def pytest_collection_modifyitems(config, items): + """Select this run's shard; without data, data tests skip unless mandatory.""" shard = config.getoption("shard", default=None) if shard is not None: this_shard, shards = shard @@ -90,19 +239,36 @@ def pytest_collection_modifyitems(config, items): if deselected: config.hook.pytest_deselected(items=deselected) - has_default_dataset = bool(os.environ.get("POLICYENGINE_UK_DEFAULT_DATASET")) - if has_default_dataset: + if TEST_DATA_PLAN.action != "skip": return + skip = pytest.mark.skip(reason=TEST_DATA_PLAN.reason) + for item in items: + if needs_data(item): + item.add_marker(skip) + - skip_microsimulation = pytest.mark.skip( - reason=( - "Requires POLICYENGINE_UK_DEFAULT_DATASET or HUGGING_FACE_TOKEN " - "for microsimulation dataset access" +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + """Fail data tests in mandatory mode without data; verify pinned bytes. + + A failure here is reported as an error in the test's setup, so the run + fails rather than passing with the test unexecuted. The hook runs before + pytest's own skip handling, so a skip marker on a data test cannot turn + missing data into a skip. + """ + if not needs_data(item): + return + plan = TEST_DATA_PLAN + if plan.action == "fail": + pytest.fail( + f"Mandatory-data mode: {plan.reason}. This test needs the certified " + "dataset and must not be skipped.", + pytrace=False, ) - ) - for item in items: - if "microsimulation" in item.keywords: - item.add_marker(skip_microsimulation) + if plan.action == "run" and plan.pinned and not runs_on_unpinned_data(item): + problem = verify_certified_dataset() + if problem: + pytest.fail(problem, pytrace=False) # Garbage collection. Most of the suite's time is in the property tests (the 51 diff --git a/policyengine_uk/tests/microsimulation/reforms_config.yaml b/policyengine_uk/tests/microsimulation/reforms_config.yaml index b13ec23de1..0d43f6b6fa 100644 --- a/policyengine_uk/tests/microsimulation/reforms_config.yaml +++ b/policyengine_uk/tests/microsimulation/reforms_config.yaml @@ -2,38 +2,218 @@ # These expected impacts are derived from current model output and are used to # catch unintended behavior changes. They are not external fiscal estimates # unless a reform entry is explicitly annotated otherwise. +# +# Each impact is the change in weighted gov_balance in 2029, in £bn, on the +# dataset below, which must be the one tests/certified_dataset.yaml pins. +# `receipts` keeps every earlier value with the model and data that produced +# it, so a change of dataset and a change of model stay separable. Tests ran +# on whatever enhanced FRS was latest until 2026-04-05, then on uk-data 1.40.3 +# (enhanced_frs_2023_24), from 2026-10-11 on the certified 1.58.0 pair, and +# from 2026-10-15 on the certified 1.58.1 pair. The 2026-10-11 receipts on +# 1.40.3 show how much of each move is the data rather than the model. +# Main fixtures without recorded model or dataset metadata retain their +# original expected_impact/tolerance/parameters format in receipts. +dataset: + data_version: 1.58.1 + path: enhanced_frs_2024_25.h5 + revision: 2526edbbb687a3b54dbb11272e5389e4b62d2ac2 + sha256: 056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784 reforms: - name: Raise basic rate by 1pp - expected_impact: 7.8 + expected_impact: 7.1 parameters: gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + receipts: + - impact: 7.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.34.5 (a1832dc77, #1205, 2025-07-11) + - impact: 7.346 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 7.132 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 7.118 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: 7.115 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: 7.115 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise higher rate by 1pp - expected_impact: 4.8 + expected_impact: 4.5 parameters: gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + receipts: + - impact: 4.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.1 (fa6eb613e, #1231, 2025-07-15) + - impact: 3.961 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 4.609 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 4.572 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: 4.490 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: 4.490 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise personal allowance by ~800GBP/year expected_impact: -4.2 parameters: gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + receipts: + - impact: -4.2 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: -4.305 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -4.135 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -4.183 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: -4.195 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: -4.195 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise child benefit by 25GBP/week per additional child - expected_impact: -1.2 + expected_impact: -1.4 parameters: gov.hmrc.child_benefit.amount.additional: 25 + receipts: + - impact: -1.2 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.0 (bda6d2ddf, #1228, 2025-07-14) + - impact: -1.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -1.428 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -1.428 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: -1.442 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: -1.442 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Reduce Universal Credit taper rate to 20% - expected_impact: -18.1 + expected_impact: -22.0 tolerance: 3.0 parameters: gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + receipts: + - impact: -18.1 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.78.0 (e4be118a7, #1572, 2026-04-11) + - impact: -18.354 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -21.266 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -21.749 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: -21.956 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: -21.956 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise Class 1 main employee NICs rate to 10% - expected_impact: 13.0 + expected_impact: 12.5 parameters: gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + receipts: + - impact: 13.0 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: 13.263 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 12.566 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 12.464 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: 12.463 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: 12.463 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise VAT standard rate by 2pp - expected_impact: 17.3 + expected_impact: 27.6 tolerance: 1.5 parameters: gov.hmrc.vat.standard_rate: 0.22 + # The move from 21.3 is the data: with the same model (2.126.8) the costing + # is 62% higher on 1.58.0 than on 1.40.3. 1.58.1 on 2.132.6 is 60% above + # 1.40.3 on 2.126.8, but those are different models, so only the 62% shows + # the data's share. VAT is linear in recorded full-rate consumption, and + # the historical 0.38 VAT coverage factor was set on older data. The + # #1996/#2187 replacement's main fixture is retained below in its format. + receipts: + - impact: 21.3 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.75.3 (bc59ba950, #1540, 2026-03-29) + - name: Raise VAT standard rate by 2pp + expected_impact: 17.3 + tolerance: 1.5 + parameters: + gov.hmrc.vat.standard_rate: 0.22 + source_commit: 98ab77093b0f48e4cb50677c2c9403a5533631f9 + - impact: 21.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 34.478 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 34.041 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: 34.041 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: 27.581 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) - name: Raise additional rate by 3pp - expected_impact: 4.4 + expected_impact: 3.7 parameters: gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + receipts: + - impact: 4.4 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.35.0 (88bdf190b, #1207, 2025-07-11) + - impact: 3.993 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 3.756 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 3.595 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.132.6 (main 1a31ef1fd, 2026-10-15) + - impact: 3.656 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.6 (merged 7efaaff16, main 62a0117f9, 2026-10-18) + - impact: 3.656 + data: 1.58.1 enhanced_frs_2024_25 + model: 2.135.7 (merged 7717438bf, main 7ad002f54, 2026-10-18) diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md new file mode 100644 index 0000000000..2dbafc68f0 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -0,0 +1,143 @@ +# Aggregate impact report + +This tool shows what a change does to national numbers. It runs the model on +the certified dataset for two checkouts (a pull request's base and its head) +and compares them. Each side runs every year from 2025-26 to 2031-32 as a full +microsimulation. + +The report covers: +- receipts by tax head; +- spending and caseloads by benefit; +- poverty; +- inequality; +- deciles; +- the standard reform costings. + +The `Aggregate impact` workflow (`.github/workflows/aggregate-impact.yaml`) +runs it on every pull request and keeps one comment on the PR up to date. + +## Data protection + +The certified dataset is licensed survey microdata. Only aggregates may be +published: `result.json`, the comment and the comparison summary. `measure` +also writes household-level detail for the decile comparison to +`private/households.npz`. Never upload, attach, commit or share anything +under `private/`, and never put the dataset in a shared cache. The workflow +measures both sides in one job so that this detail stays on the runner, and +`test_aggregate_impact_workflow.py` fails if the workflow could publish it. +Check details give counts and weighted totals, never one household's value. + +Small cells (`privacy.py`): + +- A figure from fewer than 10 records is published as suppressed. +- A total that holds one is rebuilt from its other components and marked †. +- Poverty, inequality and deciles are withheld while HBAI income holds one. +- A reform's figures for a year are withheld when they would come from a + small cell. + +`privacy.py` also states the threat model. + +## Running it + +```bash +export HUGGING_FACE_TOKEN=... # read access to policyengine-uk-data-private +uv run python -m policyengine_uk.utils.aggregate_impact measure --label head \ + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head +# ... the same for a base checkout, into out/base, then: +uv run python -m policyengine_uk.utils.aggregate_impact compare \ + --base out/base --head out/head --markdown comment.md --summary summary.json +``` + +`measure` downloads the pinned file if it isn't cached, checks its SHA-256, and +writes `result.json` (figures, checks, provenance) and +`private/households.npz` (household detail for the decile comparison). If it crashes, it still writes a +`result.json` recording the crash, so the comparison can report it. + +To measure an older checkout with this version of the tool, copy this +directory under another name and run it in that checkout's environment, as CI +does. The tool imports only its own modules and `policyengine_uk`'s public +`Microsimulation`: + +```bash +cp -R policyengine_uk/utils/aggregate_impact /tmp/tool/pe_uk_aggregate_impact +cd ../base-checkout && PYTHONPATH=/tmp/tool .venv/bin/python -m pe_uk_aggregate_impact measure ... +``` + +For use from other code (qualification in policyengine.py), see `api.py`: +`compute_budget_metrics(sim, years=...)` returns a long-format report. + +## What fails and what is flagged + +The comparison **fails** in these cases: +- the head crashes; +- base and head ran on different dataset bytes; +- a figure is not finite; +- the head breaks a check that the base passes. + +A check that fails on both sides is reported as pre-existing and does not +fail the job; `--strict` makes it fail. Moves above the thresholds in +`report_spec.yaml` are **flagged** for review, never failed, because intended +policy changes move numbers. + +The checks are: +- **closure:** for each model total (`gov_tax`, `household_tax`, + `pre_budget_change_household_tax`, `gov_spending`, `household_benefits`), the + components in the model's own list are exactly the partition's heads and + declared double counts; +- **identities**, checked household by household: + - the heads and declared double counts add up to each total; + - `gov_balance` equals `gov_tax` minus `gov_spending`; + - household net income equals market income plus benefits, less taxes and + pension contributions; +- **counted once:** each component is raised by £1 per unit, and every head + of its totals is read household by household. Its own head must rise by + exactly £1, no other tax head may move, no other benefit may rise (means + tests may claw the rise back), and the model's own total must move by what + its heads moved. A head that breaks one of these is measured again with + every other head held at baseline, and fails only where it still breaks + it; a response carried by another head (Incapacity Benefit lowers Carer's + Allowance, which Universal Credit counts as income) is reported as + mediated. A declared double count must still be contained in its head; +- **signs, bounds and finiteness**, and **determinism:** a rebuilt + simulation reproduces the baseline exactly; +- **inert heads:** a head that is zero for every unit is a warning, unless the + partition marks it `zero_at_baseline: expected`; +- **reform responsiveness:** each standard reform must move the variable it + targets in the stated direction, by at least its stated minimum, in every + year. Each minimum is a fifth of the reform's smallest yearly effect on the + certified pair, rounded down to two significant figures (the basis is in + `standard_reforms.yaml`). After a change that legitimately moves a reform's + effect, measure the new pair and run + `python -m policyengine_uk.utils.aggregate_impact calibrate ...` + to get the new `smallest_effect` and `minimum`. + +## The standard reforms + +`standard_reforms.yaml` lists the reforms costed on each side: the eight +reform-cost fixtures, plus a 10% rise in the Universal Credit standard +allowance and a 10% rise in the State Pension. Each says what it must move +(`responds`). A reform that the model ignores, or that moves its target the +wrong way, fails `responsive:` instead of passing as a small number. + +A relative reform (`scale`) is built from each side's own baseline: every +listed parameter gets that model year's baseline value times the factor, for +each year of the horizon. Pass `--reforms` to cost a different file, or +`--no-reforms` to cost none. + +## The fiscal partition + +`policyengine_uk/data/fiscal_partition.yaml` divides each total into heads, +each measured by one variable. Its header defines the fields. Each side of a +comparison reads its own partition, so a change to the model's totals and the +matching change to the partition go in the same pull request. + +When you add a component to one of the model's lists, add a head for it, or +the closure check fails. When the model stops counting something twice, +remove its `model_double_counts` entry and give it a head, or the +counted-once check fails as stale. + +## Years + +Model year 2026 is fiscal year 2026-27. Single-year datasets are projected +through 2032 (`extend_single_year_dataset`, since #2376), so every year of the +report's horizon, 2025-26 to 2031-32, is projected the same way. diff --git a/policyengine_uk/utils/aggregate_impact/__init__.py b/policyengine_uk/utils/aggregate_impact/__init__.py new file mode 100644 index 0000000000..e8c17a26e6 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__init__.py @@ -0,0 +1,14 @@ +"""Base-versus-head aggregate impact report on a certified dataset. + +The report measures one model checkout at a time (``measure``), on a dataset +whose bytes are verified against a pinned SHA-256, and then compares two +measurements (``compare``). A pull request's CI measures its base and its head +on the same dataset and posts the comparison as a comment. + +This package must only import ``policyengine_uk`` through its public +simulation API, and must import its own modules relatively: CI runs the pull +request head's copy of it against the base checkout's model. +""" + +TOOL_SCHEMA = "policyengine-uk-aggregate-impact/1" +COMMENT_MARKER = "" diff --git a/policyengine_uk/utils/aggregate_impact/__main__.py b/policyengine_uk/utils/aggregate_impact/__main__.py new file mode 100644 index 0000000000..7dce63fa60 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__main__.py @@ -0,0 +1,230 @@ +"""Command line: ``measure`` one checkout, then ``compare`` two measurements. + + python -m policyengine_uk.utils.aggregate_impact measure --label head \\ + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head + python -m policyengine_uk.utils.aggregate_impact compare \\ + --base out/base --head out/head --markdown comment.md --summary summary.json + +``measure`` exits 1 if the measurement crashed, after writing a result that +records the crash; ``compare`` exits 1 when the change fails the report. + + python -m policyengine_uk.utils.aggregate_impact calibrate \ + out/2.130.1/result.json out/head/result.json + +prints each standard reform's smallest yearly effect across the given +measurements and the minimum standard_reforms.yaml's calibration rule sets. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import traceback +from pathlib import Path + +from . import TOOL_SCHEMA + + +def _sha256(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def cmd_measure(args) -> int: + from .dataset import load_pin, materialise, verify_file + from .measure import write_result + + provenance = { + "git_sha": args.git_sha, + "git_ref": args.git_ref, + "tool": str(Path(__file__).resolve().parent), + } + try: + from .measure import measure + from .partition import default_partition_path, load_partition + from .spec import DEFAULT_SPEC_PATH, load_spec + + spec_path = Path(args.spec) if args.spec else DEFAULT_SPEC_PATH + spec = load_spec(spec_path, years=args.years) + provenance["spec_sha256"] = _sha256(spec_path) + + pin = load_pin(args.pin) + if args.dataset_file: + dataset_path = Path(args.dataset_file) + verified = verify_file(dataset_path, pin.sha256) + else: + dataset_path = materialise(pin) + verified = pin.sha256 + provenance["dataset"] = {**pin.as_dict(), "verified_sha256": verified} + + if args.partition: + partition_path, partition_source = Path(args.partition), "given" + else: + partition_path = default_partition_path() + partition_source = "measured model" + if partition_path is None: + if not args.fallback_partition: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition; " + "pass --fallback-partition" + ) + partition_path, partition_source = ( + Path(args.fallback_partition), + "fallback", + ) + partition = load_partition(partition_path) + provenance["partition_source"] = partition_source + + from .reforms import DEFAULT_REFORMS_PATH, load_reforms + + reforms = [] + if not args.no_reforms: + reforms_path = Path(args.reforms) if args.reforms else DEFAULT_REFORMS_PATH + reforms = load_reforms(reforms_path) + provenance["reforms_sha256"] = _sha256(reforms_path) + + from policyengine_uk import Microsimulation + + sim = Microsimulation(dataset=str(dataset_path)) + dataset_object = sim.dataset + + def factory(reform=None): + return Microsimulation(dataset=dataset_object, reform=reform or None) + + result, households = measure( + sim, + factory, + partition, + spec, + label=args.label, + reforms=reforms, + run_guard=not args.no_guard, + provenance=provenance, + ) + write_result(args.out, result, households) + print( + f"{args.label}: measured {len(result['metrics'])} figures in {result['timings']['total']}s" + ) + return 0 + except Exception: # noqa: BLE001 - recorded for the comparison, then reported + crash = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "label": args.label, + "provenance": provenance, + "error": traceback.format_exc(), + } + write_result(args.out, crash) + print(crash["error"], file=sys.stderr) + return 1 + + +def cmd_compare(args) -> int: + from .compare import compare + from .measure import read_result + from .render import render + from .spec import DEFAULT_SPEC_PATH, load_spec + + spec = load_spec(Path(args.spec) if args.spec else DEFAULT_SPEC_PATH) + base, base_households = read_result(args.base) if args.base else (None, None) + head, head_households = read_result(args.head) + summary = compare( + base, + head, + spec, + base_households=base_households, + head_households=head_households, + strict=args.strict, + ) + markdown = render(summary, spec, context={"run_url": args.run_url}) + if args.markdown: + Path(args.markdown).write_text(markdown) + if args.summary: + with open(args.summary, "w") as f: + json.dump(summary, f, indent=1, allow_nan=True, default=str) + print(f"aggregate impact: {summary['status']}") + for failure in summary["failures"]: + print(f"FAIL: {failure}") + return 1 if summary["status"] == "fail" else 0 + + +def cmd_calibrate(args) -> int: + import yaml + + from .reforms import DEFAULT_REFORMS_PATH, calibrate, parse_reforms + + path = Path(args.reforms or DEFAULT_REFORMS_PATH) + raw = yaml.safe_load(path.read_text()) + fraction = float((raw.get("calibration") or {}).get("fraction") or args.fraction) + results = [json.loads(Path(p).read_text()) for p in args.results] + reforms = parse_reforms({"reforms": raw.get("reforms")}) + for reform_id, row in calibrate(reforms, results, fraction).items(): + print( + f"{reform_id}: smallest_effect {row['smallest_effect']:.0f} " + f"({row['year']}), minimum {row['minimum']:.0f}" + ) + return 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk.utils.aggregate_impact" + ) + sub = parser.add_subparsers(dest="command", required=True) + + m = sub.add_parser("measure", help="measure one model checkout") + m.add_argument("--label", required=True) + m.add_argument("--pin", required=True, help="dataset pin YAML") + m.add_argument("--dataset-file", help="local copy of the pinned file (verified)") + m.add_argument("--partition", help="fiscal partition YAML to use") + m.add_argument( + "--fallback-partition", + help="partition to use if the measured model ships none", + ) + m.add_argument("--spec", help="report spec YAML") + m.add_argument("--years", help="override the horizon, e.g. 2025-2031") + m.add_argument( + "--reforms", + help="reform YAML to cost instead of the standard set (standard_reforms.yaml)", + ) + m.add_argument("--no-reforms", action="store_true", help="cost no reforms") + m.add_argument("--git-sha", default="") + m.add_argument("--git-ref", default="") + m.add_argument( + "--no-guard", action="store_true", help="skip the counted-once check" + ) + m.add_argument("--out", required=True) + m.set_defaults(func=cmd_measure) + + c = sub.add_parser("compare", help="compare a base and a head measurement") + c.add_argument("--base") + c.add_argument("--head", required=True) + c.add_argument("--spec") + c.add_argument("--markdown") + c.add_argument("--summary") + c.add_argument("--run-url", default="") + c.add_argument( + "--strict", action="store_true", help="fail on checks failing on both sides" + ) + c.set_defaults(func=cmd_compare) + + k = sub.add_parser( + "calibrate", help="recalibrate the reform responsiveness minimums" + ) + k.add_argument("results", nargs="+", help="measurement result.json files") + k.add_argument("--reforms", help="reform YAML (default standard_reforms.yaml)") + k.add_argument( + "--fraction", + type=float, + default=0.2, + help="used when the reform YAML declares no calibration.fraction", + ) + k.set_defaults(func=cmd_calibrate) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py new file mode 100644 index 0000000000..f9c0bcc575 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -0,0 +1,166 @@ +"""In-process entry point for qualification (policyengine.py, workstream W4). + +``compute_budget_metrics`` measures one simulation and returns a long-format +report: one row per metric and year, plus the identity and partition checks. +Two pairs are comparable when their reports share ``metrics_version`` (the +same tool code, and the same effective spec, horizon and reforms), whatever +model each pair runs; each model's own +fiscal partition is read from the measured package, and its digest is +``partition_version``. + +To hold the definitions fixed across models, copy this package directory +from one policyengine-uk commit and import the copy under another name: it +imports only its own modules and policyengine_uk's public simulation API. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Callable, Optional, Sequence + +from . import TOOL_SCHEMA +from .measure import measure +from .partition import Partition, default_partition_path, load_partition +from .reforms import load_reforms +from .spec import DEFAULT_SPEC_PATH, Spec, load_spec + +KIND_BY_SECTION = { + "receipts": "receipt", + "spending": "spending", + "caseloads": "caseload", + "poverty": "poverty", + "poverty_count": "poverty_count", + "inequality": "inequality", + "deciles": "decile", + "balance": "balance", + "model_totals": "model_total", + "double_counts": "double_count", + "household_totals": "household_total", + "population": "population", + "reforms": "reform", + "reform_targets": "reform_target", +} +PARTITIONED_SECTIONS = {"receipts", "spending", "caseloads"} + + +def _spec_for_years(spec: Spec, years: Optional[Sequence[int]]) -> Spec: + if not years: + return spec + years = sorted(int(y) for y in years) + guard_year = spec.guard_year if spec.guard_year in years else years[0] + return dataclasses.replace( + spec, first_year=years[0], last_year=years[-1], guard_year=guard_year + ) + + +def to_report(result: dict) -> dict: + """Flatten a measurement into the long-format qualification report.""" + provenance = result.get("provenance") or {} + report = { + "schema": TOOL_SCHEMA + "/report", + "status": result.get("status"), + "metrics_version": provenance.get("metrics_version"), + "partition_version": (result.get("partition") or {}).get("sha256"), + "years": result.get("years"), + "metrics": [], + "identities": [], + } + if result.get("status") != "ok": + report["error"] = result.get("error") + return report + for metric in result["metrics"]: + section = metric["section"] + head_id = ( + metric["id"].split(".", 1)[1] if section in PARTITIONED_SECTIONS else None + ) + for year, value in metric["values"].items(): + report["metrics"].append( + { + "metric_id": metric["id"], + "year": int(year), + # None when the figure comes from fewer than 10 records. + "value": value, + "suppressed": year in metric.get("suppressed", ()), + # The suppressed components this total is rebuilt without + # that year (measure.rebuild_protected); compare totals at + # equal exclusion. + "excludes": sorted((metric.get("excludes") or {}).get(year) or {}), + # The sign each enters the total with (+1 or -1), so two + # additive totals with different exclusions can be lined + # up from published figures (compare._aligned). + "exclude_signs": dict( + sorted(((metric.get("excludes") or {}).get(year) or {}).items()) + ), + "unit": metric["kind"], + "kind": KIND_BY_SECTION.get(section, section), + "partition": head_id, + "label": metric["label"], + } + ) + for check in result["checks"]: + report["identities"].append( + { + "identity_id": check["id"], + "year": None, + "status": check["status"], + "severity": check["severity"], + # The years the check failed in ("" for a check with no year), + # so a consumer can tell a newly failing year from an old one. + "failing_scopes": sorted(check.get("failing") or {}), + "detail": "; ".join(check["details"]), + } + ) + return report + + +def compute_budget_metrics( + sim, + factory: Optional[Callable] = None, + *, + years: Optional[Sequence[int]] = None, + partition: Optional[Partition] = None, + spec: Optional[Spec] = None, + reforms: Optional[Sequence] = None, + run_guard: bool = True, + label: str = "candidate", + provenance: Optional[dict] = None, +) -> dict: + """Measure ``sim`` (a fresh ``Microsimulation``) and return the report. + + ``factory(reform=None)`` builds another simulation on the same data; by + default it rebuilds ``type(sim)`` from ``sim.dataset``. ``reforms`` + defaults to the standard set (``standard_reforms.yaml``), each costed and + checked for responsiveness; pass ``[]`` to cost none. A failed identity + has status ``fail``; ``warn`` rows (a head that is zero everywhere) are + observations, not failures. + """ + spec = _spec_for_years(spec or load_spec(DEFAULT_SPEC_PATH), years) + if partition is None: + path = default_partition_path() + if path is None: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition" + ) + partition = load_partition(path) + if factory is None: + dataset = sim.dataset + simulation_class = type(sim) + + def factory(reform=None): + return simulation_class(dataset=dataset, reform=reform or None) + + if reforms is None: + reforms = load_reforms() + provenance = dict(provenance or {}) + result, _ = measure( + sim, + factory, + partition, + spec, + label=label, + reforms=reforms, + run_guard=run_guard, + provenance=provenance, + ) + return to_report(result) diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py new file mode 100644 index 0000000000..09a6521a0a --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -0,0 +1,657 @@ +"""Compare a base and a head measurement. + +The comparison fails when either side did not run, when they ran on +different data, when the head has a non-finite figure, when a check the base +ran is missing from the head, or when the head fails a check in a scope (a +year) where the base passes it. It reports, without failing, checks that +fail in the same scopes on both sides (pre-existing) and moves above the +flag thresholds. +""" + +from __future__ import annotations + +import math +from typing import Iterable, Mapping, Optional + +import numpy as np + +from . import TOOL_SCHEMA +from .privacy import MIN_CELL, small +from .spec import Spec, fiscal_year_label + +# Failing these on the head blocks the change whatever the base did: a +# non-finite or crashed result cannot be compared at all. +ALWAYS_BLOCKING_PREFIXES = ("finite:", "weights:") + +# Every head measurement must have run these. +REQUIRED_CHECKS = ( + "closure", + "finite:metrics", + "weights:person", + "weights:benunit", + "weights:household", +) + +FLAG_KIND_BY_SECTION = { + "reforms": "reform_gbp", + "reform_targets": "reform_gbp", +} + +DECILES = tuple(range(1, 11)) + + +def _metric_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {m["id"]: m for m in result.get("metrics", [])} + + +def _check_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {c["id"]: c for c in result.get("checks", [])} + + +def _finite(value) -> bool: + return ( + value is not None and isinstance(value, (int, float)) and math.isfinite(value) + ) + + +def classify_check( + base_status: Optional[str], + head_status: Optional[str], + blocking: bool, + base_failing: Iterable[str] = (), + head_failing: Iterable[str] = (), +) -> str: + """How a check's outcome on the head relates to the base. + + ``base_failing`` and ``head_failing`` are the scopes (years) each side + failed in. A failure is pre-existing only if the base fails in every + scope the head fails in; a newly failing year is a regression. + + Returns one of: ``pass``, ``regression``, ``pre_existing``, ``fixed``, + ``new_warning``, ``warning``, ``skipped``, ``removed``. + """ + if head_status is None: + return "removed" + if head_status == "skipped": + return "skipped" + if head_status == "pass": + return "fixed" if base_status in ("fail", "warn") else "pass" + if head_status == "warn": + return "warning" if base_status == "warn" else "new_warning" + # head failed + if blocking or base_status != "fail": + return "regression" + if set(head_failing) - set(base_failing): + return "regression" + return "pre_existing" + + +def _excludes(metric: Optional[Mapping], key: str) -> dict: + """The suppressed components a total leaves out in ``key``, with their + signs.""" + return dict(((metric or {}).get("excludes") or {}).get(key) or {}) + + +def _additive(metric_id: str, section: str) -> bool: + """Whether a metric is a plain sum of heads in pounds, so heads can be + taken out of it by subtraction. Decile means (per household) and the HBAI + total (reset at zero) are not (Sol re-review 6 of #2335).""" + return section in ("model_totals", "balance") or metric_id == ( + "household.household_net_income" + ) + + +def _aligned(value, metric_excludes: Mapping, union: Mapping, metrics: Mapping, key): + """``value`` with every head in ``union`` taken out, using published heads. + + A head excluded on the other side only is published on this side, so it + can be taken out here from its published value: the two sides then leave + out the same heads, and their difference is the published heads' change. + """ + for head_id, sign in union.items(): + if head_id in metric_excludes: + continue + published = _published(metrics, head_id, key) + if not _finite(published): + return None + value = value - sign * published + return value + + +def _published(metrics: Mapping, metric_id: str, key: str): + metric = metrics.get(metric_id) + return None if metric is None else metric["values"].get(key) + + +def compare_metrics(base: Mapping, head: Mapping, spec: Spec, years) -> tuple: + rows, flags = [], [] + ids = list(dict.fromkeys(list(head) + list(base))) + for metric_id in ids: + head_metric, base_metric = head.get(metric_id), base.get(metric_id) + template = head_metric or base_metric + row = { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "group": template.get("group"), + "kind": template["kind"], + "unit": template.get("unit"), + "values": {}, + # A figure measured on one side only is a change, never "no change". + "changed": not (head_metric and base_metric), + "only_in": None + if head_metric and base_metric + else ("head" if head_metric else "base"), + } + kind = FLAG_KIND_BY_SECTION.get(template["section"], template["kind"]) + threshold = spec.threshold(kind) + for year in years: + key = str(year) + b = base_metric["values"].get(key) if base_metric else None + h = head_metric["values"].get(key) if head_metric else None + # A figure from too few records is published as None, and a total + # containing one is rebuilt without it (privacy.py, + # measure.rebuild_protected). + b_hidden = key in (base_metric or {}).get("suppressed", ()) + h_hidden = key in (head_metric or {}).get("suppressed", ()) + b_ex, h_ex = _excludes(base_metric, key), _excludes(head_metric, key) + union = {**b_ex, **h_ex} + if b_hidden or h_hidden: + # A head from fewer than 10 records: its change is too. + delta = None + elif not (_finite(b) and _finite(h)): + delta = None + elif b_ex == h_ex: + delta = h - b + elif not _additive(metric_id, template["section"]): + # Different exclusions can't be lined up in a mean or a + # clipped total. + delta = None + else: + # The sides leave out different components: take the union + # out of both, from published values only. + b_al = _aligned(b, b_ex, union, base, key) + h_al = _aligned(h, h_ex, union, head, key) + delta = None if b_al is None or h_al is None else h_al - b_al + row["values"][key] = { + "base": b, + "head": h, + "delta": delta, + "base_suppressed": b_hidden, + "head_suppressed": h_hidden, + # The suppressed components this total leaves out, on either + # side. + "excludes": sorted(union), + # No change can be shown: a side is suppressed, or the sides + # leave out different components and can't be lined up. Not + # "no change" (Sol re-review 7 of #2335). + "withheld": delta is None + and bool(base_metric and head_metric) + and (b_hidden or h_hidden or (_finite(b) and _finite(h))), + } + if (delta is not None and delta != 0) or row["values"][key]["withheld"]: + row["changed"] = True + if ( + delta is not None + and threshold is not None + and _finite(b) + and threshold.exceeded(b, b + delta) + ): + flags.append( + { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "kind": template["kind"], + "year": year, + "base": b, + "head": h, + "delta": delta, + "limit": threshold.limit(b), + } + ) + rows.append(row) + return rows, flags + + +def _match(base_ids: np.ndarray, head_ids: np.ndarray) -> Optional[np.ndarray]: + """Indices into head arrays that line up with base households, or None. + + The two sides must hold exactly the same households, each once. + """ + base_ids, head_ids = np.asarray(base_ids), np.asarray(head_ids) + if base_ids.shape != head_ids.shape: + return None + if np.array_equal(base_ids, head_ids): + if len(np.unique(base_ids)) != len(base_ids): + return None + return np.arange(len(head_ids)) + if len(np.unique(base_ids)) != len(base_ids) or len(np.unique(head_ids)) != len( + head_ids + ): + return None + order = np.argsort(head_ids, kind="stable") + positions = np.clip(np.searchsorted(head_ids[order], base_ids), 0, len(order) - 1) + matched = order[positions] + if not np.array_equal(head_ids[matched], base_ids): + return None + return matched + + +def decile_impacts( + base_households: Mapping, head_households: Mapping, years, withheld=() +) -> tuple: + """Household-matched income changes within the base run's deciles. + + For each year and decile d of the base run: the mean change in the income + variable per household and the change relative to the decile's base + income, both weighted by the base household weights; plus the shares of + people (household weight times household size) whose household gains or + loses more than £1 a year. + + Small cells (privacy.py, Sol re-review 3 of #2335): + + - A row where fewer than ``MIN_CELL`` households with weight change at + all publishes no change: its mean change times the decile's weight + would give those households' change back. + - The income each side records leaves out any component suppressed that + year (``measure.rebuild_protected``), so no row carries a figure from + fewer than ``MIN_CELL`` records; a year whose sides leave out different + ones is withheld, with a note. + - No row publishes its households' weight or base income, and changes + are given to the precision the comment shows (£1 and 0.1%). + """ + impacts, notes = {}, [] + for year in years: + key = str(year) + b, h = base_households.get(key), head_households.get(key) + if key in {str(y) for y in withheld}: + notes.append( + f"{fiscal_year_label(year)}: decile changes are withheld: households " + f"are ranked on an income holding a figure from fewer than {MIN_CELL} " + "records" + ) + continue + if b is None or h is None: + notes.append(f"{fiscal_year_label(year)}: household detail missing") + continue + matched = _match(np.asarray(b["household_id"]), np.asarray(h["household_id"])) + if matched is None: + notes.append(f"{fiscal_year_label(year)}: base and head households differ") + continue + b_ex = sorted(str(x) for x in np.asarray(b.get("excludes", [])).ravel()) + h_ex = sorted(str(x) for x in np.asarray(h.get("excludes", [])).ravel()) + if b_ex != h_ex: + # Each side's income leaves out its own suppressed components; + # with different sets, a household's change would mix + # definitions (Sol re-review 6 of #2335). + notes.append( + f"{fiscal_year_label(year)}: a figure from fewer than {MIN_CELL} " + "records is suppressed on one side only, so decile changes are " + "withheld" + ) + continue + weight = np.asarray(b["household_weight"], dtype=np.float64) + head_weight = np.asarray(h["household_weight"], dtype=np.float64)[matched] + if not np.array_equal(weight, head_weight): + notes.append( + f"{fiscal_year_label(year)}: household weights changed; decile " + "changes use the base weights" + ) + people = np.asarray(b["household_count_people"], dtype=np.float64) + decile = np.asarray(b["decile"]) + base_income = np.asarray(b["income"], dtype=np.float64) + change = np.asarray(h["income"], dtype=np.float64)[matched] - base_income + rows = [] + for d in DECILES + ("all",): + mask = np.ones_like(decile, dtype=bool) if d == "all" else decile == d + w = weight[mask] + total_weight = float(w.sum()) + base_total = float(np.sum(w * base_income[mask])) + change_total = float(np.sum(w * change[mask])) + person_weight = w * people[mask] + person_total = float(person_weight.sum()) + changed = int(np.count_nonzero((np.abs(change[mask]) > 0.005) & (w > 0))) + if small(changed): + rows.append( + { + "decile": d, + "suppressed": True, + "mean_change": None, + "relative_change": None, + "share_gaining": None, + "share_losing": None, + } + ) + continue + rows.append( + { + "decile": d, + "suppressed": False, + # Whether any household with weight changed: published + # only for rows of 10 or more changed households, or none. + "any_change": changed > 0, + "mean_change": round(change_total / total_weight) + if total_weight + else float("nan"), + "relative_change": round(change_total / base_total, 3) + if base_total + else float("nan"), + "share_gaining": round( + float(np.sum(person_weight * (change[mask] > 1))) + / person_total, + 3, + ) + if person_total + else float("nan"), + "share_losing": round( + float(np.sum(person_weight * (change[mask] < -1))) + / person_total, + 3, + ) + if person_total + else float("nan"), + } + ) + outside = ~np.isin(decile, DECILES) & (weight > 0) + excluded = float(np.sum(weight[outside])) + if excluded: + how_many = ( + f"fewer than {MIN_CELL} households" + if small(int(outside.sum())) + else f"{excluded:,.0f} households" + ) + notes.append( + f"{fiscal_year_label(year)}: {how_many} with negative income have " + "no decile and appear only in 'all'" + ) + impacts[key] = rows + return impacts, notes + + +def _runtime_differences(base: Mapping, head: Mapping) -> list: + def listing(result): + return { + line.split("==")[0]: line + for line in (result.get("runtime") or {}).get("distributions", []) + } + + b, h = listing(base), listing(head) + differences = [] + for name in sorted(set(b) | set(h)): + if name == "policyengine-uk": + continue + if b.get(name) != h.get(name): + differences.append( + f"{b.get(name, f'{name} absent')} -> {h.get(name, f'{name} absent')}" + ) + for field in ("python", "machine"): + bv, hv = ( + (base.get("runtime") or {}).get(field), + (head.get("runtime") or {}).get(field), + ) + if bv != hv: + differences.append(f"{field}: {bv} -> {hv}") + return differences + + +def compare( + base: Optional[Mapping], + head: Optional[Mapping], + spec: Spec, + base_households: Optional[Mapping] = None, + head_households: Optional[Mapping] = None, + strict: bool = False, + require_base: bool = True, +) -> dict: + """Compare two measurements. + + With ``require_base`` (the default) a missing or crashed base fails the + comparison: without it there is nothing to show the change against. + """ + failures, warnings = [], [] + if head and head.get("status") == "skipped": + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "skipped", + "failures": [], + "warnings": [f"Not measured: {head.get('reason', 'no reason recorded')}"], + "years": spec.years, + "checks": [], + "rows": [], + "flags": [], + "deciles": {}, + "decile_notes": [], + "decile_flags": [], + "runtime_differences": [], + "base": _side_summary(base), + "head": _side_summary(head), + } + head_ok = bool(head) and head.get("status") == "ok" + base_ok = bool(base) and base.get("status") == "ok" + if not head: + failures.append( + "The head measurement produced no result; see the measure job's log." + ) + elif not head_ok: + failures.append( + "The head measurement crashed: " + _last_line(head.get("error")) + ) + base_problem = None + if not base: + base_problem = "The base measurement produced no result" + elif not base_ok: + base_problem = "The base measurement crashed: " + _last_line(base.get("error")) + if base_problem and require_base: + failures.append(base_problem + "; the change cannot be compared without it.") + elif base_problem: + warnings.append(base_problem + "; the head's own checks decide.") + + years = (head or base or {}).get("years") or spec.years + if head_ok and base_ok: + if base.get("years") != head.get("years"): + failures.append( + f"Base and head measured different years ({base.get('years')} vs {head.get('years')})." + ) + b_sha = (base.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + h_sha = (head.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + if not b_sha or not h_sha or b_sha != h_sha: + failures.append( + f"Base and head did not run on the same verified dataset ({b_sha} vs {h_sha})." + ) + if (base.get("partition") or {}).get("sha256") != ( + head.get("partition") or {} + ).get("sha256"): + warnings.append( + "This change edits the fiscal partition; review the partition diff alongside the moves it explains." + ) + runtime_differences = _runtime_differences(base, head) + if runtime_differences: + warnings.append( + "Base and head ran on different runtimes, so some changes may come from dependencies: " + + "; ".join(runtime_differences[:10]) + ) + else: + runtime_differences = [] + + base_checks, head_checks = _check_map(base), _check_map(head) + same_partition = ( + base_ok + and head_ok + and (base.get("partition") or {}).get("sha256") + == (head.get("partition") or {}).get("sha256") + ) + checks = [] + if head_ok: + missing = [c for c in REQUIRED_CHECKS if c not in head_checks] + if missing: + failures.append( + "The head measurement did not run: " + ", ".join(missing) + "." + ) + for check_id in dict.fromkeys(list(head_checks) + list(base_checks)): + b, h = base_checks.get(check_id), head_checks.get(check_id) + blocking = check_id.startswith(ALWAYS_BLOCKING_PREFIXES) or not base_ok + classification = classify_check( + b["status"] if b else None, + h["status"] if h else None, + blocking, + (b or {}).get("failing", {}), + (h or {}).get("failing", {}), + ) + if strict and classification == "pre_existing": + classification = "regression" + if classification == "skipped" and b and b["status"] != "skipped": + warnings.append( + f"The check {check_id} no longer applies on the head: " + + "; ".join((h or {}).get("details", [])[:1]) + ) + if classification == "removed" and head_ok: + # A check can disappear with the head it checked; otherwise its + # disappearance would let a failure pass unseen. + if same_partition: + failures.append( + f"The check {check_id} ran on the base but not on the head." + ) + else: + warnings.append( + f"The check {check_id} ran on the base but not on the head " + "(the partition changed)." + ) + template = h or b + checks.append( + { + "id": check_id, + "label": template["label"], + "section": template.get("section", ""), + "base_status": b["status"] if b else None, + "head_status": h["status"] if h else None, + "classification": classification, + "head_details": (h or {}).get("details", []), + "base_details": (b or {}).get("details", []), + } + ) + regressions = [c for c in checks if c["classification"] == "regression"] + for check in regressions: + failures.append( + f"{check['label']} ({check['id']}): " + "; ".join(check["head_details"][:3]) + ) + + rows, flags = compare_metrics(_metric_map(base), _metric_map(head), spec, years) + excluded = sorted( + { + head_id + for m in (head or {}).get("metrics", []) + (base or {}).get("metrics", []) + for heads in (m.get("excludes") or {}).values() + for head_id in heads + } + ) + if excluded: + warnings.append( + "In the years a figure from fewer than 10 records is suppressed (" + + ", ".join(excluded) + + "), every total containing it, and the decile incomes, are rebuilt " + "from their other components, so no published total includes it. " + "Those totals are marked †. A change is withheld where the two sides " + "leave out different figures and can't be lined up." + ) + non_finite = [] + for row in rows: + if row["only_in"] == "base" and head_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the base only." + ) + elif row["only_in"] == "head" and base_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the head only." + ) + if row["only_in"] != "base": + non_finite += [ + f"{row['id']} {fiscal_year_label(int(year))}" + for year, cell in row["values"].items() + if not _finite(cell["head"]) and not cell["head_suppressed"] + ] + if non_finite and head_ok: + more = f" and {len(non_finite) - 10} more" if len(non_finite) > 10 else "" + failures.append( + "The head has figures that are missing or not finite: " + + "; ".join(non_finite[:10]) + + more + + "." + ) + + deciles, decile_notes, decile_flags = {}, [], [] + # Years a side ranks households on an income holding a suppressed figure: + # it publishes no decile figure and keeps no household detail + # (measure.measure_year). + withheld = [ + year + for year in years + for side in (base, head) + if str(year) + in (_metric_map(side).get("deciles.mean_income.1") or {}).get("suppressed", ()) + ] + if head_ok and base_ok and ((base_households and head_households) or withheld): + # With every year withheld there is no household detail at all, and + # the notes say why (Sol re-review 8 of #2335). + deciles, decile_notes = decile_impacts( + base_households or {}, head_households or {}, years, withheld + ) + threshold = spec.threshold("decile_relative") + for year, decile_rows in deciles.items(): + for row in decile_rows: + rel = row["relative_change"] + if ( + threshold is not None + and rel is not None + and _finite(rel) + and abs(rel) > threshold.limit(0) + ): + decile_flags.append({"year": int(year), **row}) + + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "fail" if failures else "pass", + "failures": failures, + "warnings": warnings, + "years": years, + "checks": checks, + "rows": rows, + "flags": flags, + "deciles": deciles, + "decile_notes": decile_notes, + "decile_flags": decile_flags, + "runtime_differences": runtime_differences, + "base": _side_summary(base), + "head": _side_summary(head), + } + + +def _last_line(text) -> str: + lines = [line for line in str(text or "").strip().splitlines() if line.strip()] + return lines[-1] if lines else "no error recorded" + + +def _side_summary(result: Optional[Mapping]) -> Optional[dict]: + if not result: + return None + return { + "status": result.get("status"), + "label": result.get("label"), + "provenance": result.get("provenance"), + "runtime": { + k: v + for k, v in (result.get("runtime") or {}).items() + if k != "distributions" + }, + "partition": { + k: (result.get("partition") or {}).get(k) for k in ("source", "sha256") + }, + "guard": result.get("guard"), + "reforms": result.get("reforms"), + "timings": result.get("timings"), + "error": result.get("error"), + } diff --git a/policyengine_uk/utils/aggregate_impact/dataset.py b/policyengine_uk/utils/aggregate_impact/dataset.py new file mode 100644 index 0000000000..bdd32173bf --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/dataset.py @@ -0,0 +1,164 @@ +"""Pinned dataset selection: load a pin, fetch its file, verify its bytes.""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Union + +import yaml + +_HEX40 = re.compile(r"^[0-9a-f]{40}$") +_HEX64 = re.compile(r"^[0-9a-f]{64}$") +_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") + + +class DatasetPinError(ValueError): + """The pin file is malformed or names a mutable revision.""" + + +class DatasetMismatchError(RuntimeError): + """A dataset file's bytes do not match the pinned SHA-256.""" + + +@dataclass(frozen=True) +class DatasetPin: + """An immutable reference to one dataset file on the Hugging Face Hub.""" + + repo_id: str + path: str + revision: str + sha256: str + data_package: str = "" + data_version: str = "" + hf_tag: Optional[str] = None + size_bytes: Optional[int] = None + certification: Mapping[str, str] = field(default_factory=dict) + + @property + def uri(self) -> str: + """The ``hf://`` reference ``policyengine_uk.Simulation`` accepts.""" + return f"hf://{self.repo_id}/{self.path}@{self.revision}" + + def problems(self) -> list[str]: + """Every reason this pin is not an immutable, verifiable reference.""" + problems = [] + if not _REPO_ID.match(self.repo_id or ""): + problems.append(f"repo_id {self.repo_id!r} is not 'owner/name'") + if not self.path or "@" in self.path or self.path.startswith("/"): + problems.append(f"path {self.path!r} is not a relative file path") + if not _HEX40.match(self.revision or ""): + problems.append( + f"revision {self.revision!r} is not a 40-character commit hash; " + "tags and branches can move" + ) + if not _HEX64.match(self.sha256 or ""): + problems.append(f"sha256 {self.sha256!r} is not a lowercase hex digest") + if self.size_bytes is not None and self.size_bytes <= 0: + problems.append(f"size_bytes {self.size_bytes!r} is not positive") + return problems + + def describe(self) -> str: + version = f"{self.data_package} {self.data_version}".strip() + return ( + f"{self.path} ({version}, commit {self.revision[:8]}, " + f"sha256 {self.sha256[:12]})" + ) + + def as_dict(self) -> dict: + return { + "uri": self.uri, + "repo_id": self.repo_id, + "path": self.path, + "revision": self.revision, + "sha256": self.sha256, + "data_package": self.data_package, + "data_version": self.data_version, + "hf_tag": self.hf_tag, + "size_bytes": self.size_bytes, + "certification": dict(self.certification), + } + + +def load_pin(path: Union[str, os.PathLike]) -> DatasetPin: + """Read a pin file and reject it unless it is immutable and complete.""" + with open(path) as f: + raw = yaml.safe_load(f) or {} + known = { + "repo_id", + "path", + "revision", + "sha256", + "data_package", + "data_version", + "hf_tag", + "size_bytes", + "certification", + } + unknown = sorted(set(raw) - known) + if unknown: + raise DatasetPinError(f"{path}: unknown keys {unknown}") + pin = DatasetPin( + repo_id=str(raw.get("repo_id", "")), + path=str(raw.get("path", "")), + revision=str(raw.get("revision", "")), + sha256=str(raw.get("sha256", "")), + data_package=str(raw.get("data_package", "")), + data_version=str(raw.get("data_version", "")), + hf_tag=None if raw.get("hf_tag") is None else str(raw["hf_tag"]), + size_bytes=None if raw.get("size_bytes") is None else int(raw["size_bytes"]), + certification={ + str(k): str(v) for k, v in (raw.get("certification") or {}).items() + }, + ) + problems = pin.problems() + if problems: + raise DatasetPinError(f"{path}: " + "; ".join(problems)) + return pin + + +def sha256_file(path: Union[str, os.PathLike], chunk_size: int = 1 << 20) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_file(path: Union[str, os.PathLike], expected_sha256: str) -> str: + """Return the file's SHA-256, raising if it differs from the expected one.""" + actual = sha256_file(path) + if actual != expected_sha256: + raise DatasetMismatchError( + f"{path} has sha256 {actual}, but the pin requires {expected_sha256}" + ) + return actual + + +def materialise( + pin: DatasetPin, + token: Optional[str] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, +) -> Path: + """Download the pinned file (or reuse the cached copy) and verify it. + + The token defaults to ``HUGGING_FACE_TOKEN``. The Hugging Face cache is + keyed by commit, so a verified file is never downloaded twice. + """ + from huggingface_hub import hf_hub_download + + if token is None: + token = os.environ.get("HUGGING_FACE_TOKEN") or None + local = hf_hub_download( + repo_id=pin.repo_id, + filename=pin.path, + revision=pin.revision, + repo_type="model", + token=token, + cache_dir=cache_dir, + ) + verify_file(local, pin.sha256) + return Path(local) diff --git a/policyengine_uk/utils/aggregate_impact/guard.py b/policyengine_uk/utils/aggregate_impact/guard.py new file mode 100644 index 0000000000..67e05c854d --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/guard.py @@ -0,0 +1,699 @@ +"""The counted-once check: no head silently contains another head's component. + +For each component of a partition total, the check adds 1 to every unit +where the component is non-zero (to every unit, if it is zero everywhere), +recomputes every head of the totals it belongs to, and reads each head's +change household by household (see ``judge_total``): + +- the component's own head (itself, or the head a declared double count is + contained in) must rise by exactly what was added; +- no other tax head may move, and no other benefit may rise. Means tests may + lower other benefits when one rises (Pension Credit falls as State Pension + rises), but a benefit that rises with another either contains it or pays + more to people with more income. Households are judged one by one, so a + fall elsewhere cannot offset a rise; +- a head that declares a read of the component's head (``reads`` in the + partition), and any benefit when the component was zero for everyone (so + that perturbing it makes new recipients, whom passporting can reach), may + respond, but not with a slope of exactly 1. The check adds 2 as well and + fails any household where such a head rose between the two points by + exactly what was added; a step such as a passport moves it by the same + amount at both, so its slope is 0; +- the model's own total must move, household by household, by what its + heads moved, plus the addition again for a declared double count. This + catches a total whose formula adds a component twice even while that + component is zero; +- a head that breaks one of the rules above, and a head that may respond + whose slope is neither 0 nor 1, is measured again twice: with every other + head held at its baseline value, and with every other head held at its + value after the first point while the component moves on to the second. + It fails in households where either still shows the response (a move, a + rise, a slope of 1). A head can respond to the component through another + head without containing it: Incapacity Benefit reduces Carer's Allowance + (overlapping benefits), and Universal Credit, which in the model counts + Carer's Allowance as income but not Incapacity Benefit (issue #2424), + rises by what Carer's Allowance lost. Holding the other heads removes + that route. The second hold keeps a head's own response where another + head gates it: in ``h = c * (b > 0)`` with ``b = 40 * (c > 0)``, holding + ``b`` at its baseline switches ``h`` off, but holding it at its perturbed + value keeps ``h``'s slope of 1. A head that contains another head is still + found when that other head is the one perturbed. Responses through other + heads are reported as ``mediated``, with the other heads that moved in + those households: candidates for the route, not each shown to carry it. + +What it cannot see: a head that adds a component back while tapering it away +pound for pound (``max(0, a - c) + c``) does not move at the margin, so it is +indistinguishable from a flat benefit that ignores the component. Such a +head counts the component twice wherever the taper binds; only reading its +formula, or a test that sets the component's level, can find it. Likewise a +benefit that contains the component while, in every household perturbed, a +fall carried by another head offsets or outweighs its rise shows no rise and +is not measured again; in any household without that route its rise shows. + +The check reuses one simulation. It takes its snapshot of supplied inputs +only after discarding formula results, so a value some formula computed +earlier is never frozen as an input. Between perturbations it discards every +formula result, and restores any input it overrode; at the end it recomputes +the baseline and requires it to match the first computation exactly. +""" + +from __future__ import annotations + +from typing import Iterable, Mapping, Optional + +import numpy as np + +from .partition import Partition +from .privacy import count_text + + +def _raw(sim, variable, period): + try: + result = sim.calculate(variable, period, unweighted=True) + except TypeError: + result = sim.calculate(variable, period) + return np.asarray(getattr(result, "values", result)) + + +def discard_formula_results(sim) -> bool: + """Drop every calculated value, keeping supplied inputs. False if unsupported.""" + reset = getattr(sim, "_invalidate_all_caches", None) + if reset is None: + return False + reset() + return True + + +def snapshot_inputs(sim, variables) -> dict: + """Copy the supplied input arrays of ``variables``. + + Formula results are discarded first: a holder's known periods include + calculated values, and restoring one of those as an input would freeze + it for every later perturbation. + """ + discard_formula_results(sim) + snapshot = {} + for variable in variables: + if variable not in sim.tax_benefit_system.variables: + continue + holder = sim.get_holder(variable) + periods = holder.get_known_periods() + if periods: + snapshot[variable] = [ + (period, np.array(holder.get_array(period), copy=True)) + for period in periods + ] + return snapshot + + +def _restore(sim, variable, snapshot): + sim.delete_arrays(variable) + for period, values in snapshot.get(variable, []): + sim.set_input(variable, period, values) + + +def _months(year): + from policyengine_core.periods import period as make_period + + return make_period(str(year)).get_subperiods("month") + + +def _household(sim, values, entity): + values = np.asarray(values, dtype=np.float64) + if entity == "household": + return values + return np.asarray(sim.map_result(values, entity, "household"), dtype=np.float64) + + +def _head_arrays(sim, year, variables) -> dict: + """Each variable's values summed to households.""" + definitions = sim.tax_benefit_system.variables + return { + v: _household(sim, _raw(sim, v, year), definitions[v].entity.key) + for v in variables + } + + +# A household's change counts as "no change", or as "exactly what was added", +# within this many pounds plus this share of the head's own value: the model +# stores values as float32. +ABSOLUTE_SLACK_GBP = 0.02 +RELATIVE_SLACK = 2e-6 + + +def _slack(scale: np.ndarray) -> np.ndarray: + return ABSOLUTE_SLACK_GBP + RELATIVE_SLACK * np.abs(scale) + + +def _problem(total, head, kind, where, text, weights, perturbed): + """One failing (total, head, kind), with the households it fails in. + + ``where`` and ``text`` are kept for the direct-effect step and dropped + before the result is reported. + """ + # Only households with weight count towards a published count. + count = int(np.count_nonzero(where & (weights > 0))) + base = float(np.sum(weights * perturbed)) + share = float(np.sum(weights * where)) / base if base else 0.0 + return { + "total": total, + "head": head, + "kind": kind, + "households": count, + "share": share, + "message": f"{total}: {text} in {count_text(count, 'households')}", + "where": where, + "text": text, + } + + +def judge_total( + *, + total: str, + is_tax: bool, + component: str, + own_variable: str, + declared_double_count: bool, + zero_baseline: bool, + weights: np.ndarray, + expected: np.ndarray, + head_changes: Mapping[str, np.ndarray], + head_scales: Mapping[str, np.ndarray], + model_change: Optional[np.ndarray], + model_scale: Optional[np.ndarray], + allowed_readers: Iterable[str], + second_changes: Optional[Mapping[str, np.ndarray]] = None, +) -> dict: + """Apply the counted-once rules, household by household, to one total. + + ``head_changes`` maps each head variable of the total to its change by + household when 1 per unit was added to ``component`` (``expected`` by + household); ``second_changes`` gives the changes when 2 per unit were + added, where they were measured. + + - The component's own head must change by exactly ``expected`` in every + household. + - A tax head may not move in any household; a spending head may not rise + in any household. + - A head that declares a read of the component's head, and any spending + head when the component is zero for everyone, may respond, except one + for one: a household where the head's slope between the two points is + exactly what was added contains the component. + - The model's own total must change, in every household, by what its + heads changed plus ``expected`` again for a declared double count. + + Returns per-unit responses (for reports) and a list of problems, each a + mapping with ``total``, ``head``, ``kind``, ``households`` (count), + ``share`` (of the weight perturbed), ``message`` and the households' + mask ``where``. + """ + added = float(np.sum(weights * expected)) + allowed = set(allowed_readers) + perturbed = expected > 0 + responses, problems, screens, own_problem = {}, [], [], None + + def problem(head, kind, where, text): + return _problem(total, head, kind, where, text, weights, perturbed) + + def containment(head): + return f"{head} rises one for one with {component}, so it contains it," + + for variable, change in head_changes.items(): + responses[variable] = float(np.sum(weights * change)) / added if added else 0.0 + slack = _slack(head_scales[variable]) + if variable == own_variable: + off = np.abs(change - expected) > slack + if off.any(): + own_problem = problem( + variable, + "own_head", + off, + f"{variable} does not move by exactly what was added to {component}", + ) + continue + reader = variable in allowed or (zero_baseline and not is_tax) + if reader: + if second_changes is not None: + # The slope between the two points: 1 for a head that contains + # the component, 0 for a step such as a passport, whatever + # else moved the head at the first point. + slope = second_changes[variable] - change + containing = perturbed & (np.abs(slope - expected) <= 2 * slack) + # Another head's response can add to a containing head's + # slope, so any other linear response is measured again with + # the other heads held. + linear = perturbed & (np.abs(slope) > 2 * slack) & ~containing + if linear.any(): + screens.append( + problem(variable, "contains", linear, containment(variable)) + ) + else: + containing = perturbed & (np.abs(change - expected) <= slack) + if containing.any(): + problems.append( + problem(variable, "contains", containing, containment(variable)) + ) + elif is_tax: + moved = np.abs(change) > slack + if moved.any(): + problems.append( + problem( + variable, + "moves", + moved, + f"{variable} moves when {component} does", + ) + ) + else: + rises = change > slack + if rises.any(): + problems.append( + problem( + variable, + "rises", + rises, + f"{variable} rises when {component} does", + ) + ) + partition_coefficient = float(sum(responses.values())) + model_coefficient = None + if model_change is not None: + model_coefficient = ( + float(np.sum(weights * model_change)) / added if added else 0.0 + ) + heads_change = sum(head_changes.values()) + if declared_double_count: + heads_change = heads_change + expected + mismatch = np.abs(model_change - heads_change) > _slack(model_scale) + if mismatch.any(): + problems.append( + problem( + total, + "model_total", + mismatch, + f"the model's {total} moves by other than its heads when {component} does", + ) + ) + return { + "responses": responses, + "coefficient": partition_coefficient, + "model_coefficient": model_coefficient, + "own_problem": own_problem, + "problems": problems, + "screens": screens, + } + + +def _perturb(sim, variable, year, baseline, added, monthly, hold=(), held_at=None): + """Add ``added`` to ``variable``, holding each of ``hold`` fixed. + + Held variables keep their values in ``held_at`` (by default the + baseline), in the same layout: an array for a yearly variable, a list + of (month, array) for a monthly one. + """ + discard_formula_results(sim) + if monthly: + for month, values in baseline[variable]: + sim.set_input(variable, month, values + added / 12) + else: + sim.set_input(variable, year, baseline[variable] + added) + held_at = baseline if held_at is None else held_at + for held in hold: + if isinstance(held_at[held], list): + for month, values in held_at[held]: + sim.set_input(held, month, values) + else: + sim.set_input(held, year, held_at[held]) + + +def _capture(sim, variables, year, layout) -> dict: + """Each variable's current values, in ``layout``'s form (see ``_perturb``).""" + out = {} + for v in variables: + if isinstance(layout[v], list): + out[v] = [ + (month, _raw(sim, v, month).astype(np.float64)) + for month, _ in layout[v] + ] + else: + out[v] = _raw(sim, v, year).astype(np.float64) + return out + + +def breaks_on_slope(kind, where, expected, slack, slope): + """Households of ``where`` whose head's slope, with the other heads held at + their values after the first point, breaks ``kind``.""" + if kind == "contains": + return where & (np.abs(slope - expected) <= 2 * slack) + if kind == "rises": + return where & (slope > 2 * slack) + if kind == "moves": + return where & (np.abs(slope) > 2 * slack) + return where + + +def still_breaks(kind, where, expected, slack, first, second=None): + """Households of ``where`` whose head breaks ``kind`` with other heads held. + + ``first`` (and ``second``, for a containment judged on a slope) are the + head's changes when the component was raised by ``expected`` (and twice + that) with every other head at its baseline value. + """ + if kind == "contains": + if second is not None: + return where & (np.abs(second - first - expected) <= 2 * slack) + return where & (np.abs(first - expected) <= slack) + if kind == "rises": + return where & (first > slack) + if kind == "moves": + return where & (np.abs(first) > slack) + return where + + +DIRECT_KINDS = ("contains", "rises", "moves") + + +def counted_once( + sim, + partition: Partition, + year: int, + tolerance: float, + snapshot: Optional[Mapping] = None, + active: Iterable[str] = (), + variables_to_check: Optional[Iterable[str]] = None, +) -> dict: + """Perturb each component (or those in ``variables_to_check``). + + ``tolerance`` is kept for reports of per-unit responses; the verdicts + themselves are household by household (see ``judge_total``). + """ + if not discard_formula_results(sim): + return { + "status": "skipped", + "reason": "this policyengine-core has no way to discard formula results", + } + snapshot = snapshot or {} + active = set(active) + definitions = sim.tax_benefit_system.variables + totals = [t for t in partition.totals if partition.heads_in(t, active)] + heads_of = {t: [h.variable for h in partition.heads_in(t, active)] for t in totals} + all_heads = list(dict.fromkeys(v for t in totals for v in heads_of[t])) + model_totals = [t for t in totals if t in definitions] + + hw = _raw(sim, "household_weight", year).astype(np.float64) + heads0 = _head_arrays(sim, year, all_heads) + model0 = {t: _raw(sim, t, year).astype(np.float64) for t in model_totals} + baseline = {} + for variable in partition.component_variables(): + if definitions[variable].definition_period == "month": + baseline[variable] = [ + (month, _raw(sim, variable, month).astype(np.float64)) + for month in _months(year) + ] + else: + baseline[variable] = _raw(sim, variable, year).astype(np.float64) + + declared = {dc.variable: dc for dc in partition.double_counts} + head_by_variable = {h.variable: h for h in partition.heads} + selected = None if variables_to_check is None else set(variables_to_check) + results, double_count_results = [], [] + for variable in partition.component_variables(): + if selected is not None and variable not in selected: + continue + definition = definitions[variable] + entity = definition.entity.key + in_totals = [t for t in totals if variable in partition.components(t, active)] + record = { + "variable": variable, + "totals": in_totals, + "coefficients": {}, + "model_coefficients": {}, + "responses": {}, + "problems": [], + "mediated": [], + "units": 0, + "status": "pass", + "detail": "", + } + if definition.value_type is not float: + record.update(status="skipped", detail="not a money variable") + results.append(record) + continue + monthly = definition.definition_period == "month" + year_values = ( + sum(values for _, values in baseline[variable]) + if monthly + else baseline[variable] + ) + nonzero = year_values != 0 + zero_baseline = not bool(nonzero.any()) + mask = np.ones_like(nonzero) if zero_baseline else nonzero + added = mask.astype(np.float64) + expected = _household(sim, added, entity) + # Perturbed units in households with weight: a zero-weight record + # adds nothing to any published figure. + record["units"] = int(round(float(np.sum(expected[hw > 0])))) + if not float(np.sum(hw * expected)) > 0: + record.update( + status="skipped", + detail="non-zero only in households with no weight", + ) + results.append(record) + continue + own = declared[variable].contained_in if variable in declared else variable + own_head = head_by_variable[own] + needed = list(dict.fromkeys(v for t in in_totals for v in heads_of[t])) + readers_of = { + total: [ + v for v in heads_of[total] if own_head.id in head_by_variable[v].reads + ] + for total in in_totals + } + + _perturb(sim, variable, year, baseline, added, monthly) + heads1 = _head_arrays(sim, year, needed) + model1 = { + t: _raw(sim, t, year).astype(np.float64) for t in in_totals if t in model0 + } + # A head that may respond (a declared reader, or any benefit when the + # component was zero for everyone) is checked for containment at a + # second point, which tells a slope of 1 from a step. + may_respond = { + v + for total in in_totals + for v in heads_of[total] + if v != own + and ( + v in readers_of[total] + or (zero_baseline and not partition.is_tax_total(total)) + ) + } + heads2 = None + if may_respond: + _perturb(sim, variable, year, baseline, 2 * added, monthly) + heads2 = _head_arrays(sim, year, sorted(may_respond)) + problems, screens = [], [] + for total in in_totals: + verdict = judge_total( + total=total, + is_tax=partition.is_tax_total(total), + component=variable, + own_variable=own, + declared_double_count=variable in declared + and total in declared[variable].totals, + zero_baseline=zero_baseline, + weights=hw, + expected=expected, + head_changes={v: heads1[v] - heads0[v] for v in heads_of[total]}, + head_scales={v: heads0[v] for v in heads_of[total]}, + model_change=( + model1[total] - model0[total] if total in model1 else None + ), + model_scale=model0.get(total), + allowed_readers=readers_of[total], + second_changes=None + if heads2 is None + else {v: heads2[v] - heads0[v] for v in heads2}, + ) + record["coefficients"][total] = verdict["coefficient"] + if verdict["model_coefficient"] is not None: + record["model_coefficients"][total] = verdict["model_coefficient"] + record["responses"][total] = { + v: r for v, r in verdict["responses"].items() if abs(r) > tolerance + } + problems += verdict["problems"] + screens += verdict["screens"] + # A declared double count whose container no longer moves is + # reported once, as a stale declaration, below. + if verdict["own_problem"] and variable not in declared: + problems.append(verdict["own_problem"]) + # Measure each head that broke a rule again with every other head + # held at its baseline value: a response carried by another head + # (the head reading that head's change) is not containment. + mediated = [] + perturbed_units = expected > 0 + for head in dict.fromkeys( + [p["head"] for p in problems if p["kind"] in DIRECT_KINDS] + + [screen["head"] for screen in screens] + ): + held = [v for v in all_heads if v not in (variable, own, head)] + _perturb(sim, variable, year, baseline, added, monthly, hold=held) + first = _head_arrays(sim, year, [head])[head] - heads0[head] + second = None + if any( + p["head"] == head and p["kind"] == "contains" + for p in problems + screens + ): + _perturb(sim, variable, year, baseline, 2 * added, monthly, hold=held) + second = _head_arrays(sim, year, [head])[head] - heads0[head] + for v in held: + _restore(sim, v, snapshot) + # Second test: hold the other heads at their values after the + # first point, and move the component on to the second. A head + # gated by another head (h = c * (b > 0), with b a passport on c) + # keeps its own slope here, though holding b at its baseline + # switched it off above. + _perturb(sim, variable, year, baseline, added, monthly) + at_first = _capture(sim, held, year, baseline) + head_first = _head_arrays(sim, year, [head])[head] + _perturb( + sim, + variable, + year, + baseline, + 2 * added, + monthly, + hold=held, + held_at=at_first, + ) + slope_held = _head_arrays(sim, year, [head])[head] - head_first + for v in held: + _restore(sim, v, snapshot) + slack = _slack(heads0[head]) + kept = [] + for p in problems: + if p["head"] != head or p["kind"] not in DIRECT_KINDS: + kept.append(p) + continue + direct = still_breaks( + p["kind"], p["where"], expected, slack, first, second + ) | breaks_on_slope(p["kind"], p["where"], expected, slack, slope_held) + cleared = p["where"] & ~direct + if cleared.any(): + # The other heads that moved there: candidates for the + # route, not each shown to carry it. + via = [ + v + for v in needed + if v not in (variable, own, head) + and bool( + ( + np.abs(heads1[v] - heads0[v])[cleared] + > _slack(heads0[v])[cleared] + ).any() + ) + ] + mediated.append( + { + "total": p["total"], + "head": head, + "kind": p["kind"], + "households": int(np.count_nonzero(cleared & (hw > 0))), + "via": via, + } + ) + if direct.any(): + kept.append( + _problem( + p["total"], + head, + p["kind"], + direct, + p["text"], + hw, + perturbed_units, + ) + ) + # A linear response that was not one for one may hide a + # containment that another head's response added to. + for screen in screens: + if screen["head"] != head: + continue + direct = still_breaks( + "contains", screen["where"], expected, slack, first, second + ) | breaks_on_slope( + "contains", screen["where"], expected, slack, slope_held + ) + if not direct.any(): + continue + same = next( + ( + k + for k in kept + if (k["total"], k["head"], k["kind"]) + == (screen["total"], head, "contains") + ), + None, + ) + if same is not None: + kept.remove(same) + direct = direct | same["where"] + kept.append( + _problem( + screen["total"], + head, + "contains", + direct, + screen["text"], + hw, + perturbed_units, + ) + ) + problems = kept + problems = [ + {k: v for k, v in p.items() if k not in ("where", "text")} for p in problems + ] + record["mediated"] = mediated + if variable in declared: + dc = declared[variable] + container_change = heads1[own] - heads0[own] + k = float(np.sum(hw * container_change)) / float(np.sum(hw * expected)) + present = not bool( + (np.abs(container_change - expected) > _slack(heads0[own])).any() + ) + double_count_results.append( + { + "variable": variable, + "contained_in": dc.contained_in, + "coefficient": k, + "issue": dc.issue, + "status": "pass" if present else "fail", + "detail": "" + if present + else ( + f"{dc.contained_in} moves by {k:.4f} per unit of {variable}: " + "the model no longer counts it twice, so remove the " + "declaration and give it its own head" + ), + } + ) + if problems: + record.update( + status="fail", + problems=problems, + detail="; ".join(p["message"] for p in problems), + ) + results.append(record) + _restore(sim, variable, snapshot) + + discard_formula_results(sim) + heads_again = _head_arrays(sim, year, all_heads) + reproduces = all(np.array_equal(heads_again[v], heads0[v]) for v in all_heads) + return { + "status": "ran", + "year": year, + "tolerance": tolerance, + "components": results, + "double_counts": double_count_results, + "reset_reproduces_baseline": bool(reproduces), + } diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py new file mode 100644 index 0000000000..802a13624d --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -0,0 +1,1603 @@ +"""Measure one model checkout on one dataset over the report horizon.""" + +from __future__ import annotations + +import dataclasses +import gc +import hashlib +import json +import platform +import re +import sys +import time +from importlib import metadata +from pathlib import Path +from typing import Callable, Mapping, Optional, Sequence + +import numpy as np + +from .privacy import MIN_CELL, count_text, small +from . import TOOL_SCHEMA +from .guard import counted_once, snapshot_inputs +from .partition import Partition, closure_problems, group_label, read_model_lists +from .reforms import StandardReform, parse_reforms, reform_parameters, responsiveness +from .spec import Spec, fiscal_year_label + +ENTITIES = ("person", "benunit", "household") + +# Model totals reported alongside the partition. Each is a household variable. +HOUSEHOLD_TOTALS = { + "household_market_income": "Household market income", + "household_net_income": "Household net income", + "hbai_household_net_income": "Household net income (HBAI definition)", +} + + +class Recorder: + """Collects metric values and check outcomes for one measurement.""" + + def __init__(self): + self.metrics: dict = {} + self.checks: dict = {} + # The year being measured; a check records which scopes it failed + # in, so a comparison can tell a newly failing year from an old one. + self.scope: str = "" + # Raw values of suppressed heads: kept off the record, used only to + # take them out of the totals that contain them. + self.hidden: dict = {} + + def metric( + self, + metric_id, + label, + section, + kind, + year, + value, + group=None, + unit=None, + records=None, + ): + """Record a figure; ``records`` is how many records it is built from. + + A figure from fewer than ``MIN_CELL`` records (but more than none) is + published as ``None`` and its year listed in ``suppressed``. + """ + record = self.metrics.setdefault( + metric_id, + { + "id": metric_id, + "label": label, + "section": section, + "group": group, + "kind": kind, + "unit": unit, + "values": {}, + }, + ) + if records is not None and small(records): + record["values"][str(year)] = None + record.setdefault("suppressed", []).append(str(year)) + # Kept for complementary protection of totals; never published. + self.hidden[(metric_id, str(year))] = float(value) + else: + record["values"][str(year)] = float(value) + + def check( + self, check_id, label, ok, detail="", severity="fail", section="", scope=None + ): + """Record a check; repeated ids (one per year) merge into one result. + + ``failing`` maps each scope the check failed in (a year, or "" for a + check that is not per year) to its detail. ``ok=None`` records that + the check did not apply (``skipped``), which is not a pass. + """ + if ok is None: + status = "skipped" + else: + status = "pass" if ok else ("warn" if severity == "warn" else "fail") + scope = self.scope if scope is None else scope + record = self.checks.get(check_id) + if record is None: + record = { + "id": check_id, + "label": label, + "section": section, + "severity": severity, + "status": status, + "details": [], + "failing": {}, + } + self.checks[check_id] = record + elif status in ("fail", "warn") and record["status"] in ("pass", "skipped"): + record["status"] = status + elif status == "pass" and record["status"] == "skipped": + record["status"] = status + if ok is None: + if detail and detail not in record["details"]: + record["details"].append(detail) + elif not ok: + record["failing"][scope] = detail + if detail: + record["details"].append(detail) + + +def array(sim, variable: str, period, map_to: Optional[str] = None) -> np.ndarray: + """A variable's raw values, without survey weights.""" + try: + result = sim.calculate(variable, period, map_to=map_to, unweighted=True) + except TypeError: + result = sim.calculate(variable, period, map_to=map_to) + return np.asarray(getattr(result, "values", result)) + + +def active_parameters(sim, partition: Partition, year) -> set: + """The ``dropped_when`` switches that are on in ``year``.""" + active = set() + for path in partition.dropped_when_parameters(): + node = sim.tax_benefit_system.parameters + try: + for part in path.split("."): + node = getattr(node, part) + if bool(node(f"{year}-01-01")): + active.add(path) + except AttributeError: + continue + return active + + +def entity_of(sim, variable: str) -> str: + return sim.tax_benefit_system.variables[variable].entity.key + + +def to_household(sim, values: np.ndarray, entity: str) -> np.ndarray: + if entity == "household": + return np.asarray(values, dtype=np.float64) + return np.asarray( + sim.map_result(np.asarray(values, dtype=np.float64), entity, "household"), + dtype=np.float64, + ) + + +def weighted_gini(values: np.ndarray, weights: np.ndarray) -> float: + """Gini coefficient of ``values`` with non-negative ``weights``.""" + values = np.asarray(values, dtype=np.float64) + weights = np.asarray(weights, dtype=np.float64) + keep = weights > 0 + values, weights = values[keep], weights[keep] + if values.size == 0: + return float("nan") + order = np.argsort(values, kind="stable") + values, weights = values[order], weights[order] + cumulative_weight = np.cumsum(weights) + cumulative_income = np.cumsum(weights * values) + total_weight, total_income = cumulative_weight[-1], cumulative_income[-1] + if total_income == 0: + return float("nan") + previous = np.concatenate([[0.0], cumulative_income[:-1]]) + area = np.sum(weights * (cumulative_income + previous)) / ( + total_weight * total_income + ) + return float(1.0 - area) + + +def slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def _digest_tool_files() -> str: + root = Path(__file__).resolve().parent + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.suffix in (".py", ".yaml") and "__pycache__" not in path.parts: + digest.update(path.relative_to(root).as_posix().encode() + b"\0") + digest.update(path.read_bytes() + b"\0") + return digest.hexdigest() + + +# Taken when this module loads, so a result is stamped with the code that +# produced it even if the files change while a measurement runs. +_TOOL_DIGEST = _digest_tool_files() + + +def tool_digest() -> str: + """SHA-256 of this package's code and spec, independent of where it lives.""" + return _TOOL_DIGEST + + +def definitions_digest(spec: Spec, reforms: Sequence) -> str: + """SHA-256 of what was actually measured: the effective spec and reforms. + + ``tool_digest`` covers the files as shipped; this covers a spec or reform + set passed in instead, and a horizon overridden at the call. + """ + parsed = { + field.name: getattr(spec, field.name) + for field in dataclasses.fields(spec) + if field.name != "raw" + } + payload = { + "spec": spec.raw, + # The parsed fields too: a spec changed with dataclasses.replace + # keeps its raw mapping. + "parsed": parsed, + "years": spec.years, + "guard_year": spec.guard_year, + "reforms": [ + { + "id": r.id, + "parameters": dict(r.parameters), + "scale": dict(r.scale), + "responds": None + if r.responds is None + else [r.responds.variable, r.responds.direction, r.responds.minimum], + } + for r in reforms + ], + } + text = json.dumps(payload, sort_keys=True, default=str) + return hashlib.sha256(text.encode()).hexdigest() + + +def runtime_identity() -> dict: + """The installed distributions and interpreter that ran the measurement.""" + distributions = sorted( + { + (d.metadata["Name"].lower().replace("_", "-"), d.version) + for d in metadata.distributions() + if d.metadata["Name"] + } + ) + listing = "\n".join(f"{name}=={version}" for name, version in distributions) + + def version(name): + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + return { + "python": sys.version.split()[0], + "platform": platform.platform(), + "machine": platform.machine(), + "policyengine_uk": version("policyengine-uk"), + "policyengine_core": version("policyengine-core"), + "numpy": version("numpy"), + "distributions": [f"{n}=={v}" for n, v in distributions], + "distributions_sha256": hashlib.sha256(listing.encode()).hexdigest(), + } + + +# Results may be published, and the dataset is licensed survey microdata. A +# check's detail names how many records it concerns and their weighted total +# only when there are at least this many; below it, the detail says "fewer +# than" and gives no amount, so it never describes one or a few records. + +# A geographic exclusivity guard treats a household value this small as zero: +# before the #2159 fix, expected_sdlt adds 0.045 to every household. +EXCLUSIVITY_ZERO_GBP = 0.05 + + +def _weighted_records(mask, weight) -> int: + """Units in ``mask`` that carry weight: a zero-weight record adds nothing. + + Sol re-review 3 of #2335: three weighted recipients and seven zero-weight + records with values are three records, not ten. + """ + return int(np.count_nonzero(np.asarray(mask) & (np.asarray(weight) > 0))) + + +def cell_detail(count: int, what: str, weighted: str) -> str: + """``"N , "``, or ``"fewer than 10 "`` for a small cell.""" + if count < MIN_CELL: + return f"fewer than {MIN_CELL} {what}" + return f"{count:,} {what}, {weighted}" + + +def _identity_check( + rec, + check_id, + label, + components_hh, + total_hh, + year, + spec, + weights, + leave_out=None, +): + """Household-level accounting identity: components add up to the total. + + The detail gives counts and weighted totals only, and nothing at all + about fewer than ``MIN_CELL`` households (see ``cell_detail``). + + ``leave_out`` marks the households holding a component suppressed this + year, which aren't checked: with a formula that scales its components, + whether, and by how much, they differ would depend on the suppressed + values (Sol re-review 7 of #2335). The verdict and the detail then come + from the other households only. + """ + difference = components_hh - total_hh + scale = np.abs(total_hh) + 1.0 + tolerance = 1.0 + spec.identity_relative_tolerance * scale + checked = np.ones_like(difference, dtype=bool) if leave_out is None else ~leave_out + bad = (np.abs(difference) > tolerance) & checked + finite = np.all(np.isfinite(difference[checked])) + ok = bool(finite and not bad.any()) + detail = "" + if not ok: + gap = float(np.nansum(weights * np.where(bad, difference, 0.0))) + detail = f"{fiscal_year_label(year)}: " + cell_detail( + _weighted_records(bad, weights), + "households differ", + f"by a weighted £{gap:,.0f} in total", + ) + if leave_out is not None: + detail += ( + f" (households holding a figure from fewer than {MIN_CELL} " + "records aren't checked this year)" + ) + rec.check(check_id, label, ok, detail, section="identities") + + +def measure_year( + sim, partition: Partition, spec: Spec, year: int, rec: Recorder, households: dict +): + tbs_variables = sim.tax_benefit_system.variables + weights = {e: array(sim, f"{e}_weight", year).astype(np.float64) for e in ENTITIES} + for entity, label in ( + ("person", "People"), + ("benunit", "Benefit units"), + ("household", "Households"), + ): + w = weights[entity] + rec.check( + f"weights:{entity}", + f"{label} weights are finite and non-negative", + bool(np.all(np.isfinite(w)) and np.all(w >= 0)), + f"{fiscal_year_label(year)}: invalid {entity} weights", + section="data", + ) + rec.metric(f"population.{entity}", label, "population", "count", year, w.sum()) + hw = weights["household"] + + head_hh = {} + for head in partition.heads: + entity = entity_of(sim, head.variable) + values = array(sim, head.variable, year).astype(np.float64) + finite = np.isfinite(values) + records = _weighted_records(values != 0, weights[entity]) + rec.check( + f"finite:{head.id}", + f"{head.label} is finite for every unit", + bool(finite.all()), + # No count of fewer than 10 units (Sol re-review 8 of #2335). + f"{fiscal_year_label(year)}: " + + count_text(int((~finite).sum()), "non-finite values"), + section="finite", + ) + hh = to_household(sim, values, entity) + head_hh[head.variable] = hh + section = "receipts" if head.kind == "tax" else "spending" + rec.metric( + f"{section}.{head.id}", + head.label, + section, + "gbp", + year, + float(np.sum(hw * hh)), + group=group_label(head.report_group), + records=records, + ) + if head.kind == "spending" and head.caseload: + rec.metric( + f"caseloads.{head.id}", + head.label, + "caseloads", + "count", + year, + float(np.sum(weights[entity] * (values > 0))), + group=group_label(head.report_group), + unit=entity, + # Suppressed with its head: a caseload of zero would say that + # none of the head's few records is positive. + records=1 + if small(records) + else _weighted_records(values > 0, weights[entity]), + ) + if head.sign == "nonnegative" and small(records): + # Whether any of so few records is negative is about them alone + # (Sol re-review 8 of #2335): not checked this year. + rec.check( + f"sign:{head.id}", + f"{head.label} is never negative", + None, + f"{fiscal_year_label(year)}: not checked, from fewer than " + f"{MIN_CELL} records", + section="signs", + ) + elif head.sign == "nonnegative": + negative = values < -0.005 + rec.check( + f"sign:{head.id}", + f"{head.label} is never negative", + not bool(negative.any()), + f"{fiscal_year_label(year)}: " + + cell_detail( + _weighted_records(negative, weights[entity]), + "units below zero", + f"a weighted £{float(np.sum(hw * np.where(hh < 0, hh, 0.0))):,.0f} in total", + ), + section="signs", + ) + if not head.zero_at_baseline: + rec.check( + f"inert:{head.id}", + f"{head.label} is not zero for every unit", + bool(np.any(values != 0)), + f"{fiscal_year_label(year)}: zero for every unit", + severity="warn", + section="coverage", + ) + + double_count_hh = {} + for dc in partition.double_counts: + values = array(sim, dc.variable, year).astype(np.float64) + hh = to_household(sim, values, entity_of(sim, dc.variable)) + double_count_hh[dc.variable] = hh + rec.metric( + f"double_counts.{dc.variable}", + f"{dc.variable}, counted again inside {dc.contained_in}", + "double_counts", + "gbp", + year, + float(np.sum(hw * hh)), + # Like a head: from fewer than 10 records it is suppressed (Sol + # re-review 7 of #2335). + records=_weighted_records( + values != 0, weights[entity_of(sim, dc.variable)] + ), + ) + hidden = suppressed_components(rec, partition, year) + # Households holding a suppressed component: no check verdict or detail + # may depend on them this year. + holding = None + if hidden: + holding = np.zeros(len(hw), dtype=bool) + for variable in hidden: + holding |= {**head_hh, **double_count_hh}[variable] != 0 + + active = active_parameters(sim, partition, year) + # Geographic exclusivity, on the data. A declared double count is netted + # out of the head that contains it, and tiny values count as zero, so the + # guard sees each nation's own tax even before the #2159 fix. + variable_of = {head.id: head.variable for head in partition.heads} + for guard in partition.overlap_guards: + if ( + not isinstance(guard, Mapping) + or guard.get("kind") != "geographic_exclusivity" + ): + continue + members = [variable_of[h] for h in guard.get("heads") or []] + nonzero = np.zeros_like(hw, dtype=int) + for variable in members: + own = head_hh[variable].copy() + for dc in partition.double_counts: + if dc.contained_in == variable: + own = own - double_count_hh[dc.variable] + nonzero = nonzero + (np.abs(own) > EXCLUSIVITY_ZERO_GBP).astype(int) + overlapping = nonzero > 1 + if holding is not None: + # A holder's verdict would depend on the suppressed figure (Sol + # re-review 8 of #2335), as for the identities. + overlapping &= ~holding + rec.check( + f"overlap_guard:{guard.get('id')}", + "At most one of " + ", ".join(members) + " is non-zero for any household", + not bool(overlapping.any()), + f"{fiscal_year_label(year)}: " + + cell_detail( + _weighted_records(overlapping, hw), + "households have more than one", + f"a weighted {float(np.sum(hw * overlapping)):,.0f}", + ) + + ( + f" (households holding a figure from fewer than {MIN_CELL} " + "records aren't checked this year)" + if holding is not None + else "" + ), + section="partition", + ) + + model_hh = {} + for total in partition.totals: + if total not in tbs_variables: + continue + hh = to_household( + sim, array(sim, total, year).astype(np.float64), entity_of(sim, total) + ) + model_hh[total] = hh + rec.metric( + f"model.{total}", total, "model_totals", "gbp", year, float(np.sum(hw * hh)) + ) + components = np.zeros_like(hh) + for head in partition.heads_in(total, active): + components = components + head_hh[head.variable] + for dc in partition.double_counts_in(total, active): + components = components + double_count_hh[dc.variable] + _identity_check( + rec, + f"identity:{total}", + f"The partition's heads and declared double counts add up to {total}", + components, + hh, + year, + spec, + hw, + leave_out=holding, + ) + + def partition_sum(total): + out = np.zeros_like(hw) + for head in partition.heads_in(total, active): + out = out + head_hh[head.variable] + return out + + receipts = partition_sum("gov_tax") + spending = partition_sum("gov_spending") + rec.metric( + "balance.receipts", + "Government receipts, each component once", + "balance", + "gbp", + year, + float(np.sum(hw * receipts)), + ) + rec.metric( + "balance.spending", + "Government spending", + "balance", + "gbp", + year, + float(np.sum(hw * spending)), + ) + rec.metric( + "balance.balance", + "Government balance, each receipt once", + "balance", + "gbp", + year, + float(np.sum(hw * (receipts - spending))), + ) + household_only = np.zeros_like(hw) + for head in partition.spending_heads: + if "household_benefits" in head.totals and "gov_spending" not in head.totals: + household_only = household_only + head_hh[head.variable] + rec.metric( + "balance.household_only_benefits", + "Benefits in household_benefits but not in gov_spending", + "balance", + "gbp", + year, + float(np.sum(hw * household_only)), + ) + + if {"gov_balance", "gov_tax", "gov_spending"} <= set(tbs_variables): + gov_balance = array(sim, "gov_balance", year).astype(np.float64) + rec.metric( + "model.gov_balance", + "gov_balance", + "model_totals", + "gbp", + year, + float(np.sum(hw * gov_balance)), + ) + _identity_check( + rec, + "identity:gov_balance", + "gov_balance equals gov_tax minus gov_spending", + model_hh.get("gov_tax", array(sim, "gov_tax", year)) + - model_hh.get("gov_spending", array(sim, "gov_spending", year)), + gov_balance, + year, + spec, + hw, + leave_out=holding, + ) + for variable, label in HOUSEHOLD_TOTALS.items(): + if variable in tbs_variables: + values = array(sim, variable, year).astype(np.float64) + excludes = None + if variable == "hbai_household_net_income" and hidden: + values, excludes = _hbai_without(sim, year, hidden, holding) + metric_id = f"household.{variable}" + if values is None: + # The suppressed components can't be taken out of this total. + _publish_suppressed(rec, metric_id, label, "household_totals", year) + continue + rec.metric( + metric_id, + label, + "household_totals", + "gbp", + year, + float(np.sum(hw * values)), + ) + if excludes: + _exclude(rec.metrics[metric_id], str(year), excludes) + if { + "household_net_income", + "household_market_income", + "household_benefits", + "household_tax", + "pension_contributions", + } <= set(tbs_variables): + pension = to_household( + sim, + array(sim, "pension_contributions", year).astype(np.float64), + entity_of(sim, "pension_contributions"), + ) + components = ( + array(sim, "household_market_income", year).astype(np.float64) + + model_hh.get("household_benefits", array(sim, "household_benefits", year)) + - model_hh.get("household_tax", array(sim, "household_tax", year)) + - pension + ) + _identity_check( + rec, + "identity:household_net_income", + "Household net income equals market income plus benefits less taxes and pension contributions", + components, + array(sim, "household_net_income", year).astype(np.float64), + year, + spec, + hw, + leave_out=holding, + ) + + pw = weights["person"] + for measure_spec in spec.poverty_measures: + try: + in_poverty = array( + sim, measure_spec["variable"], year, map_to="person" + ).astype(bool) + except Exception as error: # noqa: BLE001 - recorded, not raised + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + True, + section="coverage", + ) + for group in spec.poverty_groups: + if group.get("variable"): + try: + members = array(sim, group["variable"], year).astype(bool) + except Exception as error: # noqa: BLE001 + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + True, + section="coverage", + ) + else: + members = np.ones_like(in_poverty, dtype=bool) + population = float(np.sum(pw * members)) + poor = float(np.sum(pw * (members & in_poverty))) + rate = poor / population if population > 0 else float("nan") + rec.check( + "bounds:poverty_rates", + "Poverty rates lie between 0 and 1", + bool(0.0 <= rate <= 1.0), + f"{fiscal_year_label(year)}: {measure_spec['id']} {group['id']} is {rate}", + section="bounds", + ) + metric_id = f"{measure_spec['id']}.{group['id']}" + label = f"{measure_spec['label']}: {group['label'].lower()}" + rec.metric( + f"poverty.{metric_id}", + label, + "poverty", + "rate", + year, + rate, + group=measure_spec["label"], + ) + rec.metric( + f"poverty_count.{metric_id}", + label, + "poverty_count", + "count", + year, + poor, + group=measure_spec["label"], + ) + + people = array(sim, "household_count_people", year).astype(np.float64) + for inequality in spec.inequality: + income = array(sim, inequality["variable"], year).astype(np.float64) + rec.metric( + f"inequality.{inequality['id']}", + inequality["label"], + "inequality", + "index", + year, + weighted_gini(income, hw * people), + ) + + net_income, net_income_excludes = rebuild_protected( + rec, + partition, + active, + hidden, + {**head_hh, **double_count_hh}, + hw, + year, + market_hh=( + array(sim, "household_market_income", year).astype(np.float64) + if "household_market_income" in tbs_variables + else None + ), + pension_hh=( + to_household( + sim, + array(sim, "pension_contributions", year).astype(np.float64), + entity_of(sim, "pension_contributions"), + ) + if "pension_contributions" in tbs_variables + else None + ), + ) + decile = array(sim, spec.decile_variable, year).astype(np.int64) + income = array(sim, spec.income_variable, year).astype(np.float64) + income_excludes = {} + ranking_holds_hidden = bool(hidden) and hbai_holds_any(hidden) + if ranking_holds_hidden: + # Households are ranked into deciles on HBAI income, which holds a + # suppressed component this year: which decile its holders fall in + # depends on it, so no decile figure is published. + income = None + for d in range(1, 11): + rec.metric( + f"deciles.mean_income.{d}", + f"Decile {d}: mean {spec.income_variable} per household", + "deciles", + "gbp_household", + year, + float("nan"), + group="Own-decile levels", + records=1, # published as suppressed (privacy.py) + ) + elif net_income_excludes: + if spec.income_variable == "household_net_income" and net_income is not None: + income, income_excludes = net_income, net_income_excludes + else: + income = None # its suppressed parts can't be taken out of it + for d in range(1, 11) if income is not None else (): + in_decile = decile == d + weight = float(np.sum(hw[in_decile])) + rec.metric( + f"deciles.mean_income.{d}", + f"Decile {d}: mean {spec.income_variable} per household", + "deciles", + "gbp_household", + year, + float(np.sum(hw[in_decile] * income[in_decile])) / weight + if weight + else float("nan"), + group="Own-decile levels", + ) + if income_excludes: + _exclude( + rec.metrics[f"deciles.mean_income.{d}"], str(year), income_excludes + ) + if ranking_holds_hidden: + # Poverty and inequality are measured on HBAI income, which holds a + # suppressed component this year; a Gini or poverty count with the + # totals published could give it back (Sol re-review 6 of #2335). + for record in rec.metrics.values(): + if record["section"] in ("poverty", "poverty_count", "inequality"): + if record["values"].get(str(year)) is not None: + record["values"][str(year)] = None + record.setdefault("suppressed", []).append(str(year)) + if income is not None: + households[str(year)] = { + "household_id": array(sim, "household_id", year).astype(np.int64), + "household_weight": hw.astype(np.float64), + "household_count_people": people.astype(np.float64), + "decile": decile, + "income": income, + # The suppressed components the income leaves out: both sides + # must leave out the same ones for their decile rows to compare. + "excludes": np.array(sorted(income_excludes), dtype=str), + } + + +def _exclude(record: dict, key: str, excludes: Mapping) -> None: + """Record which suppressed components a figure leaves out in ``key``.""" + # Rebound rather than written through, which the cached-array scan + # (test_cached_arrays_not_written_in_place) would read as an array write. + record["excludes"] = {**(record.get("excludes") or {}), key: dict(excludes)} + + +def suppressed_components(rec: "Recorder", partition: Partition, year: int) -> dict: + """``{variable: (metric id, raw value)}`` for each head or declared double + count suppressed in ``year``.""" + key = str(year) + out = {} + for head in partition.heads: + metric_id = f"{'receipts' if head.kind == 'tax' else 'spending'}.{head.id}" + if (metric_id, key) in rec.hidden: + out[head.variable] = (metric_id, rec.hidden[(metric_id, key)]) + for dc in partition.double_counts: + metric_id = f"double_counts.{dc.variable}" + if (metric_id, key) in rec.hidden: + out[dc.variable] = (metric_id, rec.hidden[(metric_id, key)]) + return out + + +def rebuild_protected( + rec: "Recorder", + partition: Partition, + active, + hidden: Mapping, + component_hh: Mapping, + hw: np.ndarray, + year: int, + market_hh=None, + pension_hh=None, +) -> tuple: + """Publish each total that holds a suppressed component rebuilt without it. + + The total is rebuilt household by household from its other components, + never as the model's total less the suppressed one, so no published + figure depends on a suppressed component's values: not through a formula + that scales its components, and not through an identity's tolerance (Sol + re-reviews 5 to 7 of #2335). Each rebuilt figure is: + + - a model total: its other heads and declared double counts; + - a balance aggregate: its other heads; + - ``gov_balance``: the rebuilt ``gov_tax`` less the rebuilt + ``gov_spending``; + - household net income: market income plus the rebuilt + ``household_benefits``, less the rebuilt ``household_tax`` and pension + contributions (its own adds and subtracts). Suppressed if market + income or pension contributions aren't available. + + Its record lists the left-out components, with the sign each enters + with, under ``excludes``. Returns the rebuilt household net income per + household (None if not rebuilt) and what it leaves out, for the deciles. + """ + if not hidden: + return None, {} + key = str(year) + + def parts(total, sign=1, double_counts=True): + out = [(h.variable, sign) for h in partition.heads_in(total, active)] + if double_counts: + out += [ + (d.variable, sign) for d in partition.double_counts_in(total, active) + ] + return out + + def rebuilt(signed_parts): + """``(per-household sum of the published parts, excludes)``, or None + when no part is suppressed.""" + if not any(variable in hidden for variable, _ in signed_parts): + return None + total = np.zeros_like(hw, dtype=np.float64) + excludes = {} + for variable, sign in signed_parts: + if variable in hidden: + metric_id = hidden[variable][0] + excludes[metric_id] = excludes.get(metric_id, 0) + sign + else: + total = total + sign * component_hh[variable] + return total, {m: sign for m, sign in excludes.items() if sign} + + def publish(metric_id, result): + record = rec.metrics.get(metric_id) + if result is None or record is None or record["values"].get(key) is None: + return + values, excludes = result + record["values"][key] = float(np.sum(hw * values)) + if excludes: + _exclude(record, key, excludes) + + for total in partition.totals: + publish(f"model.{total}", rebuilt(parts(total))) + receipts = parts("gov_tax", double_counts=False) + spending = parts("gov_spending", -1, double_counts=False) + publish("balance.receipts", rebuilt(receipts)) + publish("balance.spending", rebuilt([(v, 1) for v, _ in spending])) + publish("balance.balance", rebuilt(receipts + spending)) + publish( + "balance.household_only_benefits", + rebuilt( + [ + (h.variable, 1) + for h in partition.spending_heads + if "household_benefits" in h.totals and "gov_spending" not in h.totals + ] + ), + ) + publish("model.gov_balance", rebuilt(parts("gov_tax") + parts("gov_spending", -1))) + + net = rebuilt(parts("household_benefits") + parts("household_tax", -1)) + if net is None: + return None, {} + record = rec.metrics.get("household.household_net_income") + if market_hh is None or pension_hh is None: + if record is not None and record["values"].get(key) is not None: + record["values"][key] = None + record.setdefault("suppressed", []).append(key) + return None, net[1] + income = market_hh + net[0] - pension_hh + publish("household.household_net_income", (income, net[1])) + return income, net[1] + + +def _hbai_lists(): + """The HBAI income component lists, or None if the model has none.""" + try: + from policyengine_uk.variables.household.income import ( + hbai_household_net_income as hbai, + ) + + return ( + set(hbai.HBAI_HOUSEHOLD_NET_INCOME_ADDS), + set(hbai.HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS), + ) + except (ImportError, AttributeError): + return None + + +def hbai_holds_any(hidden: Mapping) -> bool: + """Whether a suppressed component is part of HBAI income (True if unknown).""" + lists = _hbai_lists() + if lists is None: + return True + adds, subtracts = lists + return any(variable in adds or variable in subtracts for variable in hidden) + + +# How far, per household, the HBAI component lists may be from the model's +# income before the reset: half a penny, the precision of the published +# figures. The rebuilt total never uses a suppressed component, so this bounds +# only a wrong figure (a component the model leaves out under the current +# policy), never a disclosure (Sol re-review 7 of #2335). +HBAI_REPRODUCTION_GBP = 0.005 + + +def _hbai_without(sim, year, hidden, holding=None): + """The HBAI household net income without the suppressed components, or None. + + Rebuilt household by household from its component lists without the + suppressed ones, then reset at zero as HBAI does: it never uses their + values. The lists must reproduce the model's income before the reset + (``HBAI_REPRODUCTION_GBP``) for every household not in ``holding`` (those + holding a suppressed component, whose verdict would depend on it), or + the total is suppressed (None): a component the model leaves out under + the current policy (council tax when it is abolished, say) would + otherwise come back in (Sol re-reviews 6 and 7 of #2335). None also when + the model doesn't expose the income before the reset or the lists. + """ + lists = _hbai_lists() + if lists is None: + return None, None + adds, subtracts = lists + inside = { + variable: 1 if variable in adds else -1 + for variable in hidden + if variable in adds or variable in subtracts + } + values = array(sim, "hbai_household_net_income", year).astype(np.float64) + if not inside: + return values, None + name = "hbai_household_net_income_before_reset_to_zero" + variables = sim.tax_benefit_system.variables + if name not in variables: + return None, None + before = array(sim, name, year).astype(np.float64) + rebuilt = np.zeros_like(before) + without = np.zeros_like(before) + for variable, sign in [(v, 1) for v in sorted(adds)] + [ + (v, -1) for v in sorted(subtracts) + ]: + if variable in variables: + part = array(sim, variable, year, map_to="household").astype(np.float64) + rebuilt = rebuilt + sign * part + if variable not in inside: + without = without + sign * part + checked = slice(None) if holding is None else ~np.asarray(holding) + if not np.all(np.abs(rebuilt - before)[checked] <= HBAI_REPRODUCTION_GBP): + return None, None + excludes = {hidden[v][0]: sign for v, sign in inside.items()} + return np.maximum(0.0, without), excludes + + +def _publish_suppressed(rec: "Recorder", metric_id, label, section, year): + record = rec.metrics.setdefault( + metric_id, + { + "id": metric_id, + "label": label, + "section": section, + "group": None, + "kind": "gbp", + "unit": None, + "values": {}, + }, + ) + record["values"][str(year)] = None + record.setdefault("suppressed", []).append(str(year)) + + +def publishable_guard(guard: dict) -> dict: + """The counted-once result with small cells removed (privacy.py). + + A component perturbed in fewer than ``MIN_CELL`` units keeps no count, no + responses or coefficients, and no weighted shares in its problems. A + problem or mediated response in fewer than ``MIN_CELL`` households keeps + no count and no share. Messages already say "fewer than 10". + """ + out = dict(guard) + components = [] + for row in guard.get("components") or []: + row = dict(row) + few_units = small(row.get("units") or 0) + if few_units: + # How the heads responded to so few units says what those + # recipients receive (Sol re-review 3 of #2335). + row["units"] = None + for key in ("responses", "coefficients", "model_coefficients"): + if key in row: + row[key] = {} + for key in ("problems", "mediated"): + cleaned = [] + for item in row.get(key) or []: + item = dict(item) + if few_units or small(item.get("households") or 0): + item.pop("share", None) + if small(item.get("households") or 0): + item["households"] = None + cleaned.append(item) + row[key] = cleaned + components.append(row) + if "components" in guard: + out["components"] = components + # A declared double count's coefficient is its container's mean response + # across its units: for fewer than MIN_CELL units it goes too (Sol + # re-review 8 of #2335). + units = { + row.get("variable"): row.get("units") for row in guard.get("components") or [] + } + double_counts = [] + for row in guard.get("double_counts") or []: + row = dict(row) + if small(units.get(row.get("variable")) or 0): + row["coefficient"] = None + if row.get("status") != "pass": + row["detail"] = ( + f"{row['contained_in']} no longer moves one for one with " + f"{row['variable']} (fewer than {MIN_CELL} units): the model " + "no longer counts it twice, so remove the declaration and give " + "it its own head" + ) + double_counts.append(row) + if "double_counts" in guard: + out["double_counts"] = double_counts + return out + + +def _finite_metrics_check(rec: Recorder): + bad = [ + f"{m['id']} {fiscal_year_label(int(year))}" + for m in rec.metrics.values() + for year, value in m["values"].items() + if year not in m.get("suppressed", ()) and not np.isfinite(value) + ] + rec.check( + "finite:metrics", + "Every reported figure is finite", + not bad, + "; ".join(bad[:20]) + (f" and {len(bad) - 20} more" if len(bad) > 20 else ""), + section="finite", + ) + + +def weighted_total(sim, variable: str, year: int) -> float: + """The household-weighted total of a variable of any entity.""" + hw = array(sim, "household_weight", year).astype(np.float64) + values = array(sim, variable, year).astype(np.float64) + return float(np.sum(hw * to_household(sim, values, entity_of(sim, variable)))) + + +# A record's income or tax changes when it moves by more than half a penny, +# as for decile rows (compare.decile_impacts). +CHANGE_GBP = 0.005 + + +def baseline_records(sim, partition: Partition, rec: "Recorder", year: int, targets): + """What the reform checks need from the baseline run in ``year``: each + suppressed component's values, each household's gov_balance and each + target variable's values. Kept in memory only, never written.""" + return { + "hidden": { + variable: array(sim, variable, year).astype(np.float64).copy() + for variable in suppressed_components(rec, partition, year) + }, + "household_weight": array(sim, "household_weight", year) + .astype(np.float64) + .copy(), + "balance": to_household( + sim, + array(sim, "gov_balance", year).astype(np.float64), + entity_of(sim, "gov_balance"), + ).copy(), + "targets": { + variable: array(sim, variable, year).astype(np.float64).copy() + for variable in targets + }, + } + + +def reform_withheld( + reformed, year: int, baseline: Optional[Mapping], components +) -> bool: + """Whether a reform's costing and target change for ``year`` are withheld. + + They would be built from a small cell (Sol re-review 7 of #2335) when the + reform: + + - changes any record of a component suppressed in the baseline, by any + amount, even if the changes cancel nationally: they still move other + figures (the tax on one recipient's pension, say). An unchanged record + compares equal, because a rebuilt baseline reproduces the first run + exactly (``determinism:rebuilt_baseline``); noise could only withhold + more; + - leaves any other component with 1 to 9 records (``privacy.small``), + whose change would give that new small cell back; + - changes gov_balance in 1 to 9 households, or a target variable in 1 to + 9 records, by more than ``CHANGE_GBP``: the change is theirs alone, as + for a decile row. + + Withheld too when there is no baseline record for the year. + """ + if baseline is None: + return True + for variable, before in baseline["hidden"].items(): + if not np.array_equal( + array(reformed, variable, year).astype(np.float64), before + ): + return True + weights = {} + + def weight(variable): + entity = entity_of(reformed, variable) + if entity not in weights: + weights[entity] = array(reformed, f"{entity}_weight", year).astype( + np.float64 + ) + return weights[entity] + + for variable in components: + if variable in baseline["hidden"]: + continue + values = array(reformed, variable, year).astype(np.float64) + if small(_weighted_records(values != 0, weight(variable))): + return True + balance = to_household( + reformed, + array(reformed, "gov_balance", year).astype(np.float64), + entity_of(reformed, "gov_balance"), + ) + changed = np.abs(balance - baseline["balance"]) > CHANGE_GBP + if small(_weighted_records(changed, baseline["household_weight"])): + return True + for variable, before in baseline["targets"].items(): + after = array(reformed, variable, year).astype(np.float64) + if small( + _weighted_records(np.abs(after - before) > CHANGE_GBP, weight(variable)) + ): + return True + return False + + +def as_standard_reforms(reforms) -> list: + """Accept parsed reforms or raw ``{name, parameters, ...}`` mappings.""" + reforms = list(reforms or []) + if all(isinstance(r, StandardReform) for r in reforms): + return reforms + return parse_reforms({"reforms": reforms}) + + +def measure_reforms( + factory: Callable, + baseline_parameters, + baseline_balance: Mapping[int, float], + baseline_targets: Mapping[int, Mapping[str, float]], + reforms: Sequence[StandardReform], + years: Sequence[int], + rec: Recorder, + baseline: Optional[Mapping[int, Mapping]] = None, + components: Sequence[str] = (), +) -> list: + """Cost each reform against the baseline run, and check it responds. + + Records each reform's change in gov_balance by year and, where the + reform declares what it must move, the change in that variable's total + and whether it moved enough in the right direction every year. + + ``baseline`` gives, for each year, the baseline's records + (``baseline_records``), and ``components`` the partition's component + variables. A year whose figures would come from a small cell + (``reform_withheld``) publishes neither the costing nor the target + change, and its responsiveness detail gives no amount: a 10% uplift's + cost of £100 would give back a suppressed £1,000 (Sol re-reviews 6 and 7 + of #2335). Without ``baseline``, every year is withheld. + + First rebuilds an unreformed simulation the same way, which must + reproduce the baseline exactly: otherwise reform costings would include + differences that no reform caused. + """ + outcomes = [] + try: + fresh = factory(reform=None) + for year in years: + again = weighted_total(fresh, "gov_balance", year) + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + again == baseline_balance[year], + # No amounts: an exact gov_balance, less the published heads, + # would give a suppressed head back (Sol re-review 3 of #2335). + f"{fiscal_year_label(year)}: the rebuilt baseline's gov_balance " + "differs from the first run's", + section="determinism", + scope=str(year), + ) + del fresh + except Exception as error: # noqa: BLE001 + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + False, + f"{type(error).__name__}: {error}", + section="determinism", + ) + gc.collect() + for reform in reforms: + outcome = { + "id": reform.id, + "name": reform.name, + "parameters": None, + "responds": None, + "error": None, + } + target_changes = {} + hidden_years = set() + try: + parameters = reform_parameters(reform, baseline_parameters, years) + outcome["parameters"] = parameters + reformed = factory(reform=parameters) + for year in years: + withheld = reform_withheld( + reformed, year, (baseline or {}).get(year), components + ) + if withheld: + hidden_years.add(year) + # records=1 publishes the figure as suppressed (privacy.py). + records = 1 if withheld else None + rec.metric( + f"reforms.{reform.id}", + reform.name, + "reforms", + "gbp", + year, + weighted_total(reformed, "gov_balance", year) + - baseline_balance[year], + records=records, + ) + if reform.responds is not None: + variable = reform.responds.variable + change = ( + weighted_total(reformed, variable, year) + - baseline_targets[year][variable] + ) + target_changes[year] = change + rec.metric( + f"reform_targets.{reform.id}", + f"{reform.name}: change in {variable}", + "reform_targets", + "gbp", + year, + change, + records=records, + ) + del reformed + except Exception as error: # noqa: BLE001 - one reform's failure is reported, not fatal + outcome["error"] = f"{type(error).__name__}: {error}" + if reform.responds is not None: + outcome["responds"] = { + "variable": reform.responds.variable, + "direction": reform.responds.direction, + "minimum": reform.responds.minimum, + } + label = ( + f"'{reform.name}' moves {reform.responds.variable} " + f"{reform.responds.direction} by at least " + f"£{reform.responds.minimum / 1e9:,.2f}bn in every year" + ) + if outcome["error"] is not None: + for year in years: + rec.check( + f"responsive:{reform.id}", + label, + False, + f"{fiscal_year_label(year)}: the reform did not run", + section="reforms", + scope=str(year), + ) + else: + # One verdict per year, so a newly inert year is a regression + # even when another year already failed. + for year in years: + ok, detail = responsiveness( + reform.responds, {year: target_changes.get(year, float("nan"))} + ) + if year in hidden_years and not ok: + detail = ( + f"{fiscal_year_label(year)}: {reform.responds.variable} did " + "not move enough (the amount is withheld: it comes from " + f"fewer than {MIN_CELL} records)" + ) + rec.check( + f"responsive:{reform.id}", + label, + ok, + detail, + section="reforms", + scope=str(year), + ) + gc.collect() + outcomes.append(outcome) + return outcomes + + +def measure( + sim, + factory: Callable, + partition: Partition, + spec: Spec, + *, + label: str, + reforms: Sequence[Mapping] = (), + run_guard: bool = True, + provenance: Optional[Mapping] = None, + guard_variables: Optional[Sequence[str]] = None, +) -> tuple: + """Measure a simulation over the spec's horizon. + + ``sim`` is a fresh simulation on the dataset; ``factory(reform=None)`` + builds another on the same dataset. Returns ``(result, households)``. + """ + started = time.time() + timings = {} + rec = Recorder() + model_lists, list_problems = read_model_lists(partition) + problems = list_problems + closure_problems( + partition, model_lists, sim.tax_benefit_system.variables + ) + closure_label = ( + "Each model total's components are exactly the partition's heads and " + "declared double counts" + ) + rec.check("closure", closure_label, True, section="partition", scope="") + # One scope per problem: its text names the total and the variable. + for problem_text in problems: + rec.check( + "closure", + closure_label, + False, + problem_text, + section="partition", + scope=problem_text, + ) + snapshot = snapshot_inputs(sim, partition.component_variables()) + + reforms = as_standard_reforms(reforms) + target_variables = sorted({r.responds.variable for r in reforms if r.responds}) + households = {} + baseline_balance, baseline_targets, baseline = {}, {}, {} + reset = getattr(sim, "_invalidate_all_caches", None) + for year in spec.years: + t = time.time() + rec.scope = str(year) + measure_year(sim, partition, spec, year, rec, households) + rec.scope = "" + baseline_balance[year] = weighted_total(sim, "gov_balance", year) + baseline_targets[year] = { + variable: weighted_total(sim, variable, year) + for variable in target_variables + } + # So that reform figures from a small cell are withheld. + baseline[year] = baseline_records(sim, partition, rec, year, target_variables) + timings[f"year_{year}"] = round(time.time() - t, 2) + if reset is not None: + # Keep memory flat across the horizon; supplied inputs survive. + reset() + gc.collect() + + guard = {"status": "skipped", "reason": "not requested"} + if run_guard: + t = time.time() + guard = counted_once( + sim, + partition, + spec.guard_year, + spec.counted_once_tolerance, + snapshot, + active=active_parameters(sim, partition, spec.guard_year), + variables_to_check=guard_variables, + ) + timings["counted_once"] = round(time.time() - t, 2) + for result in guard.get("components", []): + check_id = f"counted_once:{result['variable']}" + label = ( + f"Raising {result['variable']} by £1 raises its own head by £1, moves no " + "other tax head and raises no other benefit" + ) + if result["status"] != "fail": + rec.check( + check_id, + label, + None if result["status"] == "skipped" else True, + result["detail"], + section="counted_once", + scope="", + ) + # One scope per failing (total, head, kind), so a comparison sees + # a head that newly contains the component, not just "still fails". + for item in result.get("problems") or []: + rec.check( + check_id, + label, + False, + item["message"], + section="counted_once", + scope=f"{item['total']}:{item['head']}:{item['kind']}", + ) + # Details from the published guard, with small cells removed. + for result in publishable_guard(guard).get("double_counts", []): + rec.check( + f"double_count_present:{result['variable']}", + f"{result['variable']} is still contained in {result['contained_in']}", + result["status"] == "pass", + result["detail"], + section="counted_once", + ) + if "reset_reproduces_baseline" in guard: + rec.check( + "counted_once:reset_reproduces_baseline", + "The counted-once check leaves the baseline unchanged", + bool(guard["reset_reproduces_baseline"]), + "recomputed baseline totals differ from the first run", + section="counted_once", + ) + rec.check( + "counted_once:ran", + "The counted-once check ran", + guard.get("status") == "ran", + guard.get("reason", ""), + severity="warn", + section="counted_once", + ) + + t = time.time() + reform_outcomes = measure_reforms( + factory, + sim.tax_benefit_system.parameters, + baseline_balance, + baseline_targets, + reforms, + spec.years, + rec, + baseline=baseline, + components=partition.component_variables(), + ) + timings["reforms"] = round(time.time() - t, 2) + for outcome in reform_outcomes: + rec.check( + f"reform_runs:{outcome['id']}", + f"The reform '{outcome['name']}' runs", + outcome["error"] is None, + outcome["error"] or "", + section="reforms", + ) + # After the reform costings, so that they are covered too. + _finite_metrics_check(rec) + timings["total"] = round(time.time() - started, 2) + + provenance = dict(provenance or {}) + provenance["tool_sha256"] = tool_digest() + provenance["definitions_sha256"] = definitions_digest(spec, reforms) + provenance["metrics_version"] = ( + f"{TOOL_SCHEMA}+{provenance['tool_sha256'][:16]}" + f"+{provenance['definitions_sha256'][:16]}" + ) + result = { + "schema": TOOL_SCHEMA, + "status": "ok", + "label": label, + "years": spec.years, + "provenance": provenance, + "runtime": runtime_identity(), + "partition": partition.as_dict(), + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + "guard": publishable_guard(guard), + "reforms": reform_outcomes, + "timings": timings, + } + return result, households + + +PRIVATE_DIRECTORY = "private" +HOUSEHOLDS_FILE = "households.npz" + + +def write_result(out_dir, result: Mapping, households: Optional[Mapping] = None): + """Write the aggregates to result.json and household detail to private/. + + ``result.json`` holds only aggregates and may be published. Everything + under ``private/`` is household-level detail from licensed microdata: it + exists for the decile comparison and must never be uploaded or shared. + """ + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + with open(out_dir / "result.json", "w") as f: + json.dump(result, f, indent=1, sort_keys=False, allow_nan=True) + if households: + arrays = { + f"{year}__{name}": values + for year, columns in households.items() + for name, values in columns.items() + } + private = out_dir / PRIVATE_DIRECTORY + private.mkdir(exist_ok=True) + np.savez_compressed(private / HOUSEHOLDS_FILE, **arrays) + + +def read_result(out_dir) -> tuple: + """Load ``(result, households)`` written by :func:`write_result`.""" + out_dir = Path(out_dir) + path = out_dir / "result.json" + if not path.exists(): + return None, None + with open(path) as f: + result = json.load(f) + households = None + npz = out_dir / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE + if npz.exists(): + households = {} + with np.load(npz) as data: + for key in data.files: + year, _, name = key.partition("__") + households.setdefault(year, {})[name] = data[key] + return result, households diff --git a/policyengine_uk/utils/aggregate_impact/partition.py b/policyengine_uk/utils/aggregate_impact/partition.py new file mode 100644 index 0000000000..91892300f5 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/partition.py @@ -0,0 +1,557 @@ +"""The fiscal partition: loading, structural validation and closure checks. + +The partition lives with the model at +``policyengine_uk/data/fiscal_partition.yaml`` (schema 1; its header defines +the fields). This module reads the YAML directly, not through any loader in +the model, so the pull request head's copy of this tool can read the base +checkout's partition. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +SUPPORTED_SCHEMA_VERSIONS = {1} +PARTITION_RELATIVE_PATH = Path("data") / "fiscal_partition.yaml" +ENTITIES = {"person", "benunit", "household"} +SIGNS = {None, "nonnegative", "any"} + +# Where each total's component list lives in the model, and which kind of +# heads it divides into. A total in the file may override these. +KNOWN_TOTALS = { + "gov_tax": ("tax", "policyengine_uk.variables.gov.gov_tax", "GOV_TAX_VARIABLES"), + "household_tax": ( + "tax", + "policyengine_uk.variables.gov.hmrc.household_tax", + "HOUSEHOLD_TAX_VARIABLES", + ), + "pre_budget_change_household_tax": ( + "tax", + "policyengine_uk.variables.contrib.policyengine.pre_budget_change_household_tax", + "PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES", + ), + "gov_spending": ( + "spending", + "policyengine_uk.variables.gov.gov_spending", + "GOV_SPENDING_VARIABLES", + ), + "household_benefits": ( + "spending", + "policyengine_uk.variables.household.income.household_benefits", + "HOUSEHOLD_BENEFIT_VARIABLES", + ), +} +GOVERNMENT_TOTALS = {"gov_tax", "gov_spending"} + + +class PartitionError(ValueError): + """The partition file is malformed or internally inconsistent.""" + + +@dataclass(frozen=True) +class TotalSpec: + name: str + kind: str # "tax" or "spending" + module: str + list_name: str + label: str = "" + dropped_when: tuple = () # ((parameter path, (variable, ...)), ...) + + +@dataclass(frozen=True) +class Head: + id: str + label: str + variable: str + totals: tuple + report_group: str + kind: str # "tax" or "spending" + entity: Optional[str] = None + obr_line: Optional[str] = None + payer: Optional[str] = None + sign: Optional[str] = None + zero_at_baseline: bool = False + caseload: bool = False + reads: tuple = () + notes: tuple = () + + +@dataclass(frozen=True) +class DoubleCount: + variable: str + contained_in: str + totals: tuple + issue: str = "" + + +@dataclass(frozen=True) +class Exclusion: + variable: str + kind: str = "" + head: str = "" + spans: tuple = () + note: str = "" + + +@dataclass(frozen=True) +class Partition: + schema_version: int + totals: Mapping[str, TotalSpec] + tax_heads: tuple + spending_heads: tuple + double_counts: tuple + exclusions: tuple + overlap_guards: tuple = () + source: str = "" + sha256: str = "" + raw: Mapping = field(default_factory=dict, compare=False, repr=False) + + @property + def heads(self) -> tuple: + return self.tax_heads + self.spending_heads + + def dropped(self, total: str, active: Iterable[str] = ()) -> set: + """Variables the total leaves out while the given parameters are true.""" + active = set(active) + spec = self.totals.get(total) + if spec is None: + return set() + return { + v + for parameter, variables in spec.dropped_when + if parameter in active + for v in variables + } + + def heads_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + h for h in self.heads if total in h.totals and h.variable not in dropped + ] + + def double_counts_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + d + for d in self.double_counts + if total in d.totals and d.variable not in dropped + ] + + def components(self, total: str, active: Iterable[str] = ()) -> list: + """The variables the model should add for ``total``, per the partition.""" + return [h.variable for h in self.heads_in(total, active)] + [ + d.variable for d in self.double_counts_in(total, active) + ] + + def component_variables(self) -> list: + """Every variable named as a head or a declared double count, once.""" + return list( + dict.fromkeys( + [h.variable for h in self.heads] + + [d.variable for d in self.double_counts] + ) + ) + + def head_by_variable(self, variable: str) -> Optional[Head]: + for head in self.heads: + if head.variable == variable: + return head + return None + + def is_tax_total(self, total: str) -> bool: + spec = self.totals.get(total) + return spec is not None and spec.kind == "tax" + + def dropped_when_parameters(self) -> list: + return list( + dict.fromkeys( + p for spec in self.totals.values() for p, _ in spec.dropped_when + ) + ) + + def as_dict(self) -> dict: + return { + "source": self.source, + "sha256": self.sha256, + "schema_version": self.schema_version, + "tax_heads": [_head_dict(h) for h in self.tax_heads], + "spending_heads": [_head_dict(h) for h in self.spending_heads], + "model_double_counts": [ + { + "variable": d.variable, + "contained_in": d.contained_in, + "totals": list(d.totals), + "issue": d.issue, + } + for d in self.double_counts + ], + "totals": { + name: { + "kind": t.kind, + "module": t.module, + "list": t.list_name, + "dropped_when": {p: list(v) for p, v in t.dropped_when}, + } + for name, t in self.totals.items() + }, + } + + +def _head_dict(head: Head) -> dict: + return { + "id": head.id, + "label": head.label, + "variable": head.variable, + "totals": list(head.totals), + "report_group": head.report_group, + "kind": head.kind, + "entity": head.entity, + "obr_line": head.obr_line, + "payer": head.payer, + "sign": head.sign, + "zero_at_baseline": head.zero_at_baseline, + "caseload": head.caseload, + "notes": list(head.notes), + } + + +def group_label(report_group: str) -> str: + """``excise_duties`` -> ``Excise duties``; labels already in prose pass through.""" + text = report_group.replace("_", " ").strip() + return text[:1].upper() + text[1:] if text else "Other" + + +def default_partition_path() -> Optional[Path]: + """The partition shipped with the installed ``policyengine_uk``, if any.""" + spec = importlib.util.find_spec("policyengine_uk") + if spec is None or not spec.submodule_search_locations: + return None + for location in spec.submodule_search_locations: + candidate = Path(location) / PARTITION_RELATIVE_PATH + if candidate.exists(): + return candidate + return None + + +def _as_tuple(value, field_name: str, where: str) -> tuple: + if value is None: + return () + if isinstance(value, str): + raise PartitionError(f"{where}: {field_name} must be a list, not a string") + return tuple(str(v) for v in value) + + +def _parse_total(name: str, raw: Mapping) -> TotalSpec: + if raw is None: + raw = {} + if not isinstance(raw, Mapping): + raise PartitionError(f"totals.{name} must be a mapping") + known = KNOWN_TOTALS.get(name) + kind = raw.get("kind") or (known[0] if known else None) + module = raw.get("module") or (known[1] if known else None) + list_name = raw.get("list") or (known[2] if known else None) + if kind not in ("tax", "spending") or not module or not list_name: + raise PartitionError( + f"totals.{name}: an unfamiliar total needs kind (tax or spending), module and list" + ) + dropped = [] + for parameter, variables in (raw.get("dropped_when") or {}).items(): + dropped.append( + (str(parameter), _as_tuple(variables, "dropped_when", f"totals.{name}")) + ) + return TotalSpec( + name=name, + kind=kind, + module=str(module), + list_name=str(list_name), + label=str(raw.get("label") or ""), + dropped_when=tuple(dropped), + ) + + +def _parse_head(raw: Mapping, kind: str, index: int) -> Head: + where = f"{kind}_heads[{index}]" + if not isinstance(raw, Mapping): + raise PartitionError(f"{where}: must be a mapping") + missing = [k for k in ("id", "label", "variable", "totals") if not raw.get(k)] + if missing: + raise PartitionError(f"{where}: missing {missing}") + zero = raw.get("zero_at_baseline") + if zero not in (None, "expected"): + raise PartitionError(f"{where}: zero_at_baseline must be 'expected' or absent") + notes = tuple( + f"{key.replace('_', ' ')}: {' '.join(str(raw[key]).split())}" + for key in ("obr_line_note", "zero_at_baseline_note", "data_note", "model_note") + if raw.get(key) + ) + reads = tuple( + str(r.get("head")) for r in (raw.get("reads") or []) if isinstance(r, Mapping) + ) + caseload = raw.get("caseload") + return Head( + id=str(raw["id"]), + label=str(raw["label"]), + variable=str(raw["variable"]), + totals=_as_tuple(raw.get("totals"), "totals", where), + report_group=str(raw.get("report_group") or "other"), + kind=kind, + entity=raw.get("entity"), + obr_line=raw.get("obr_line"), + payer=raw.get("payer"), + sign=raw.get("sign"), + zero_at_baseline=zero == "expected", + caseload=bool(caseload) if caseload is not None else kind == "spending", + reads=reads, + notes=notes, + ) + + +def parse_partition( + raw: Mapping, source: str = "", sha256: str = "" +) -> Partition: + """Build a partition from parsed YAML, raising on any inconsistency.""" + if not isinstance(raw, Mapping): + raise PartitionError(f"{source}: the partition must be a mapping") + version = raw.get("schema_version") + if version not in SUPPORTED_SCHEMA_VERSIONS: + raise PartitionError( + f"{source}: schema_version {version!r} is not one of " + f"{sorted(SUPPORTED_SCHEMA_VERSIONS)}" + ) + totals = { + str(name): _parse_total(str(name), spec) + for name, spec in (raw.get("totals") or {}).items() + } + tax_heads = tuple( + _parse_head(h, "tax", i) for i, h in enumerate(raw.get("tax_heads") or []) + ) + spending_heads = tuple( + _parse_head(h, "spending", i) + for i, h in enumerate(raw.get("spending_heads") or []) + ) + double_counts = [] + for i, dc in enumerate(raw.get("model_double_counts") or []): + where = f"model_double_counts[{i}]" + if ( + not isinstance(dc, Mapping) + or not dc.get("variable") + or not dc.get("contained_in") + ): + raise PartitionError(f"{source}: {where} needs variable and contained_in") + double_counts.append( + DoubleCount( + variable=str(dc["variable"]), + contained_in=str(dc["contained_in"]), + totals=_as_tuple(dc.get("totals"), "totals", where), + issue=str(dc.get("issue") or ""), + ) + ) + exclusions = tuple( + Exclusion( + variable=str(e.get("variable")), + kind=str(e.get("kind") or ""), + head=str(e.get("head") or ""), + spans=_as_tuple(e.get("spans"), "spans", "exclusions"), + note=str(e.get("note") or e.get("reason") or ""), + ) + for e in (raw.get("exclusions") or []) + if isinstance(e, Mapping) + ) + partition = Partition( + schema_version=version, + totals=totals, + tax_heads=tax_heads, + spending_heads=spending_heads, + double_counts=tuple(double_counts), + exclusions=exclusions, + overlap_guards=tuple(raw.get("overlap_guards") or ()), + source=source, + sha256=sha256, + raw=raw, + ) + problems = structural_problems(partition) + if problems: + raise PartitionError(f"{source}:\n- " + "\n- ".join(problems)) + return partition + + +def load_partition(path) -> Partition: + path = Path(path) + data = path.read_bytes() + return parse_partition( + yaml.safe_load(data), source=str(path), sha256=hashlib.sha256(data).hexdigest() + ) + + +def structural_problems(partition: Partition) -> list: + """Inconsistencies visible from the partition alone, without the model.""" + problems = [] + heads = partition.heads + head_ids = {h.id for h in heads} + for name, count in Counter(h.id for h in heads).items(): + if count > 1: + problems.append(f"head id {name!r} is used {count} times") + for name, count in Counter(h.variable for h in heads).items(): + if count > 1: + problems.append(f"variable {name!r} is the variable of {count} heads") + head_variables = {h.variable for h in heads} + for head in heads: + if len(set(head.totals)) != len(head.totals): + problems.append(f"head {head.id!r} lists a total twice") + for total in head.totals: + spec = partition.totals.get(total) + if spec is None: + problems.append(f"head {head.id!r} names unknown total {total!r}") + elif spec.kind != head.kind: + problems.append( + f"{head.kind} head {head.id!r} is in {total!r}, a {spec.kind} total" + ) + if head.entity is not None and head.entity not in ENTITIES: + problems.append(f"head {head.id!r} has unknown entity {head.entity!r}") + if head.sign not in SIGNS: + problems.append(f"head {head.id!r} has unknown sign {head.sign!r}") + for read in head.reads: + if read not in head_ids: + problems.append( + f"head {head.id!r} declares a read of unknown head {read!r}" + ) + for name, spec in partition.totals.items(): + listed = set(partition.components(name)) + for parameter, variables in spec.dropped_when: + for variable in variables: + if variable not in listed: + problems.append( + f"totals.{name} drops {variable!r} under {parameter}, but no head " + "or double count of that total is that variable" + ) + for dc in partition.double_counts: + if dc.variable in head_variables: + problems.append( + f"double count {dc.variable!r} is also a head; it would be counted twice" + ) + container = partition.head_by_variable(dc.contained_in) + if container is None: + problems.append( + f"double count {dc.variable!r} is contained in {dc.contained_in!r}, " + "which is not a head variable" + ) + else: + missing = [t for t in dc.totals if t not in container.totals] + if missing: + problems.append( + f"double count {dc.variable!r} is in {missing}, which its " + f"container {dc.contained_in!r} is not" + ) + for total in dc.totals: + if total not in partition.totals: + problems.append( + f"double count {dc.variable!r} names unknown total {total!r}" + ) + for name, count in Counter(d.variable for d in partition.double_counts).items(): + if count > 1: + problems.append(f"double count {name!r} is declared {count} times") + for guard in partition.overlap_guards: + if isinstance(guard, Mapping) and guard.get("kind") == "geographic_exclusivity": + named = list(guard.get("heads") or []) + if len(named) < 2: + problems.append( + f"overlap guard {guard.get('id')!r} needs at least two heads" + ) + for name in named: + if name not in head_ids: + problems.append( + f"overlap guard {guard.get('id')!r} names unknown head {name!r}" + ) + for exclusion in partition.exclusions: + if exclusion.variable in head_variables: + problems.append( + f"excluded variable {exclusion.variable!r} is also a head variable" + ) + for referenced in ([exclusion.head] if exclusion.head else []) + list( + exclusion.spans + ): + if referenced not in head_ids: + problems.append( + f"exclusion {exclusion.variable!r} refers to unknown head {referenced!r}" + ) + return problems + + +def read_model_lists(partition: Partition) -> tuple: + """Import each total's component list from the model. + + Returns ``(lists, problems)``: the lists that could be read, keyed by + total, and a description of every total whose list could not be read. + """ + lists, problems = {}, [] + for name, spec in partition.totals.items(): + try: + module = importlib.import_module(spec.module) + values = getattr(module, spec.list_name) + except (ImportError, AttributeError) as error: + problems.append( + f"{name}: cannot read {spec.module}.{spec.list_name}: {error}" + ) + continue + lists[name] = [str(v) for v in values] + return lists, problems + + +def closure_problems( + partition: Partition, + model_lists: Mapping[str, Iterable[str]], + model_variables: Optional[Mapping] = None, +) -> list: + """Differences between the partition and the model's own component lists. + + For every total, the heads and double counts naming it must be exactly + the model's list, each listed once; every named variable must exist, and + a head's declared entity must be its variable's entity. + """ + problems = [] + for total, listed in model_lists.items(): + listed = list(listed) + for variable, count in Counter(listed).items(): + if count > 1: + problems.append(f"{total}: the model lists {variable!r} {count} times") + expected, assigned = set(listed), set(partition.components(total)) + for variable in sorted(expected - assigned): + problems.append( + f"{total}: the model adds {variable!r}, which no head or declared " + "double count covers" + ) + for variable in sorted(assigned - expected): + problems.append( + f"{total}: the partition puts {variable!r} in this total, but the " + "model does not add it" + ) + if model_variables is not None: + named = partition.component_variables() + [ + d.contained_in for d in partition.double_counts + ] + for variable in sorted(set(named) - set(model_variables)): + problems.append(f"{variable!r} is not a model variable") + for head in partition.heads: + definition = ( + model_variables.get(head.variable) + if hasattr(model_variables, "get") + else None + ) + if ( + head.entity + and definition is not None + and definition.entity.key != head.entity + ): + problems.append( + f"head {head.id!r} says entity {head.entity!r}, but {head.variable} " + f"is a {definition.entity.key} variable" + ) + return problems diff --git a/policyengine_uk/utils/aggregate_impact/privacy.py b/policyengine_uk/utils/aggregate_impact/privacy.py new file mode 100644 index 0000000000..80a6c32e75 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/privacy.py @@ -0,0 +1,58 @@ +"""Small-cell rules for everything the report publishes. + +The dataset is licensed survey microdata. A figure built from fewer than +``MIN_CELL`` records can reveal one of them: a benefit total and its +caseload from one recipient give the award and the survey weight. Such +figures are published as suppressed, and descriptions of fewer than +``MIN_CELL`` units give no count or weighted share. + +A total that contains a suppressed head or double count is rebuilt that year +from its other components (``measure.rebuild_protected``), and so are the +household incomes for deciles, so no published total includes it. Checks +whose verdict would depend on its values leave out the households holding it +(identities, the HBAI reproduction, overlap guards) or aren't run that year +(the sign check of the suppressed head). A reform's costing and target change +are withheld in a year they would come from a small cell +(``measure.reform_withheld``). + +Deliberate exceptions: + +- Other heads whose formulas read the suppressed figure (income tax on a + taxable benefit, a means test) still respond to it, inside totals from 10 + or more records; exact changes between versions are published under the + d1601 A ruling below. +- The counted-once and double-count-present verdicts for a component from + fewer than 10 units depend on how other heads respond at those units. They + are kept, as the only containment tests for that component; their counts, + responses and coefficients are not published. +- Reform withholding counts households, not amounts: one household changing + by £10,000 alongside ten changing by a penny is published. That is the same + counting rule as for levels; no dominance rule is applied. + +Threat model. The report defends against disclosure through any combination +of the figures it publishes, across any number of reports, from honest +changes to the model, the data or the reforms. It doesn't defend against the +author of a pull request in this repository: GitHub runs a ``pull_request`` +workflow from the pull request's branch, so its author could read the data in +the job directly. A pull request from a fork can't: the job runs only for +branches of this repository (its ``if`` in ``aggregate-impact.yaml``), and +GitHub gives fork pull requests no secrets. + +d1601 A (Max, 2026-10-17): publish exact changes between model versions (or +datasets), as policyengine-api does; level protection stays. A change in +fewer than 10 records of a national total with 10 or more is published +exactly. Decile rows and reform figures retain their small-change withholding. +""" + +MIN_CELL = 10 +SUPPRESSED = f"fewer than {MIN_CELL} records" + + +def small(count: int) -> bool: + """A non-empty cell too small to publish.""" + return 0 < count < MIN_CELL + + +def count_text(count: int, what: str) -> str: + """``"N "``, or ``"fewer than 10 "`` for a small cell.""" + return f"fewer than {MIN_CELL} {what}" if small(count) else f"{count:,} {what}" diff --git a/policyengine_uk/utils/aggregate_impact/reforms.py b/policyengine_uk/utils/aggregate_impact/reforms.py new file mode 100644 index 0000000000..e3fd60d156 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/reforms.py @@ -0,0 +1,245 @@ +"""The standard reform set: loading, building reforms, and responsiveness.""" + +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Sequence + +import yaml + +from .spec import fiscal_year_label + +DEFAULT_REFORMS_PATH = Path(__file__).with_name("standard_reforms.yaml") +_INDEXED = re.compile(r"^(?P[^\[\]]+)\[(?P\d+)\]$") + + +class ReformSpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Responds: + """What a reform must do to the weighted total of one model variable.""" + + variable: str + direction: str # "up" or "down" + minimum: float + # The reform's smallest yearly effect, in the declared direction, on the + # pair the minimum was calibrated on (see ``calibrated_minimum``). + smallest_effect: Optional[float] = None + + +@dataclass(frozen=True) +class StandardReform: + id: str + name: str + parameters: Mapping = field(default_factory=dict) + scale: Mapping = field(default_factory=dict) + responds: Optional[Responds] = None + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def calibrated_minimum(smallest_effect: float, fraction: float) -> float: + """``fraction`` of the smallest yearly effect, rounded down to 2 significant figures.""" + amount = fraction * smallest_effect + if not amount > 0: + raise ReformSpecError( + "a calibrated minimum needs a positive effect and fraction" + ) + step = 10 ** (math.floor(math.log10(amount)) - 1) + return float(math.floor(amount / step) * step) + + +def parse_reforms(raw: Mapping) -> list: + """Parse a reform file. + + Accepts the standard schema and the reform-cost fixture schema (a + ``reforms`` list of ``name`` and ``parameters``, with no ``responds``). + + When the file declares ``calibration.fraction``, every ``responds`` must + give the ``smallest_effect`` it was calibrated on, and its ``minimum`` + must be ``calibrated_minimum(smallest_effect, fraction)``: a minimum + edited by hand, or an effect updated without its minimum, is refused. + """ + reforms, seen = [], set() + fraction = ((raw or {}).get("calibration") or {}).get("fraction") + if fraction is not None and not 0 < float(fraction) < 1: + raise ReformSpecError("calibration.fraction must lie between 0 and 1") + for index, entry in enumerate((raw or {}).get("reforms") or []): + where = f"reforms[{index}]" + if not isinstance(entry, Mapping) or not entry.get("name"): + raise ReformSpecError(f"{where}: needs a name") + parameters = dict(entry.get("parameters") or {}) + scale = {str(k): float(v) for k, v in (entry.get("scale") or {}).items()} + if not parameters and not scale: + raise ReformSpecError(f"{where}: needs parameters or scale") + for path, factor in scale.items(): + if not factor > 0: + raise ReformSpecError( + f"{where}: scale factor for {path} must be positive" + ) + responds = None + if entry.get("responds"): + r = entry["responds"] + if r.get("direction") not in ("up", "down"): + raise ReformSpecError(f"{where}: responds.direction must be up or down") + minimum = float(r.get("minimum", 0)) + if not r.get("variable") or not minimum > 0: + raise ReformSpecError( + f"{where}: responds needs a variable and a positive minimum" + ) + smallest = r.get("smallest_effect") + smallest = None if smallest is None else float(smallest) + if fraction is not None: + if smallest is None: + raise ReformSpecError( + f"{where}: the file is calibrated, so responds needs " + "the smallest_effect its minimum was set from" + ) + expected = calibrated_minimum(smallest, float(fraction)) + if minimum != expected: + raise ReformSpecError( + f"{where}: minimum {minimum:,.0f} is not {fraction} of " + f"the smallest effect {smallest:,.0f} rounded down to " + f"two significant figures ({expected:,.0f})" + ) + responds = Responds(str(r["variable"]), r["direction"], minimum, smallest) + reform_id = str(entry.get("id") or _slug(str(entry["name"]))) + if reform_id in seen: + raise ReformSpecError(f"{where}: reform id {reform_id!r} is used twice") + seen.add(reform_id) + reforms.append( + StandardReform(reform_id, str(entry["name"]), parameters, scale, responds) + ) + return reforms + + +def load_reforms(path=None) -> list: + with open(path or DEFAULT_REFORMS_PATH) as f: + return parse_reforms(yaml.safe_load(f)) + + +def calibrate( + reforms: Sequence[StandardReform], results: Sequence[Mapping], fraction: float +) -> dict: + """Each reform's smallest yearly effect across ``results``, and its minimum. + + ``results`` are measurement result.json mappings, typically one per model + version of the pair being calibrated on. An effect counts in the + reform's declared direction, so a reform that moves the wrong way in any + year or measurement cannot be calibrated. + """ + out = {} + for reform in reforms: + if reform.responds is None: + continue + sign = 1.0 if reform.responds.direction == "up" else -1.0 + effects = [] + for result in results: + metric = next( + ( + m + for m in result.get("metrics", []) + if m["id"] == f"reform_targets.{reform.id}" + ), + None, + ) + if metric is None: + raise ReformSpecError(f"{reform.id}: a result has no target change") + effects += [(sign * v, int(y)) for y, v in metric["values"].items()] + smallest, year = min(effects) + if not smallest > 0: + raise ReformSpecError( + f"{reform.id}: moves {reform.responds.variable} the wrong way or " + f"not at all in {fiscal_year_label(year)}" + ) + out[reform.id] = { + "smallest_effect": smallest, + "year": year, + "minimum": calibrated_minimum(smallest, fraction), + } + return out + + +def parameter_node(root, path: str): + """Navigate ``a.b[0].c`` from a parameter tree's root.""" + node = root + for part in path.split("."): + indexed = _INDEXED.match(part) + if indexed: + node = getattr(node, indexed["name"]).brackets[int(indexed["index"])] + else: + node = getattr(node, part) + return node + + +# Parameter metadata under which a bare year does not mean model year Y on the +# reform-dict route (it means the fiscal year from 6 April), or under which the +# value read at 1 January Y is not the value the model uses for Y. +CALENDAR_DATED_METADATA = ("preserve_calendar_dates", "fiscal_year_blend") + + +def reform_parameters( + reform: StandardReform, baseline_parameters, years: Sequence[int] +) -> dict: + """The parameter-change dict for ``Microsimulation(reform=...)``. + + A scaled parameter gets one bare-year entry per horizon year: the + baseline's value for that model year times the factor. On the + reform-dict route a bare year sets model year Y alone, and the processed + baseline's value at 1 January Y is the value the model uses for Y + (``convert_to_fiscal_year_parameters`` writes the fiscal year's value + over the calendar year). Neither holds for a parameter that keeps its + calendar dates or blends fiscal years, so scaling one is refused. + """ + changes = dict(reform.parameters) + for path, factor in reform.scale.items(): + node = parameter_node(baseline_parameters, path) + metadata = getattr(node, "metadata", None) or {} + dated = [key for key in CALENDAR_DATED_METADATA if metadata.get(key)] + if dated: + raise ReformSpecError( + f"{reform.id}: cannot scale {path}, which has {', '.join(dated)}; " + "write its values with explicit dates instead" + ) + changes[path] = { + str(year): float(node(f"{year}-01-01")) * factor for year in years + } + return changes + + +def responsiveness(responds: Responds, changes: Mapping[int, float]) -> tuple: + """Whether a reform moved its target enough, the right way, in every year. + + ``changes`` maps year to the reformed total minus the baseline total. + Returns ``(ok, detail)``; ``detail`` names the failing years. + """ + sign = 1.0 if responds.direction == "up" else -1.0 + failing = [] + for year, change in sorted(changes.items()): + moved = sign * change + if not moved >= responds.minimum: + if change == 0: + what = "did not move" + elif moved < 0: + what = f"moved the wrong way (£{change:,.0f})" + else: + what = f"moved only £{change:,.0f}" + failing.append( + f"{fiscal_year_label(int(year))}: {responds.variable} {what}" + ) + if not changes: + failing.append("no year was measured") + detail = "" + if failing: + detail = ( + f"expected {responds.variable} {responds.direction} by at least " + f"£{responds.minimum:,.0f} a year; " + "; ".join(failing) + ) + return not failing, detail diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py new file mode 100644 index 0000000000..f90e870071 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -0,0 +1,459 @@ +"""Render a comparison as a Markdown pull request comment.""" + +from __future__ import annotations + +import math +from typing import Mapping, Optional + +from . import COMMENT_MARKER +from .spec import Spec, fiscal_year_label + +SECTIONS = ( + ("receipts", "Receipts by tax head"), + ("spending", "Spending by benefit and service"), + ("caseloads", "Caseloads"), + ("balance", "Government balance, each component counted once"), + ("model_totals", "Model totals"), + ("double_counts", "Components the model counts twice"), + ("household_totals", "Household income totals"), + ("population", "Population"), + ("poverty", "Poverty rates"), + ("poverty_count", "People in poverty"), + ("inequality", "Inequality"), + ("deciles", "Mean household net income in each side's own deciles"), + ("reforms", "Standard reform costings (change in gov_balance)"), + ("reform_targets", "Standard reforms: change in each reform's target"), +) +MAX_FLAG_ROWS = 40 + + +def _finite(value) -> bool: + return isinstance(value, (int, float)) and math.isfinite(value) + + +def fmt_level(value, kind) -> str: + if not _finite(value): + return "n/a" + if kind == "gbp": + return f"£{value / 1e9:,.2f}bn" + if kind == "gbp_household": + return f"£{value:,.0f}" + if kind == "count": + return f"{value / 1e3:,.0f}k" + if kind == "rate": + return f"{100 * value:.1f}%" + return f"{value:.4f}" + + +def fmt_change(value, kind) -> str: + if value is None or not _finite(value): + return "n/a" + if value == 0: + return "0" + sign = "+" if value > 0 else "-" + size = abs(value) + if kind == "gbp": + if size >= 1e9: + return f"{sign}£{size / 1e9:,.2f}bn" + if size >= 1e5: + return f"{sign}£{size / 1e6:,.1f}m" + return f"{sign}£{size:,.0f}" + if kind == "gbp_household": + return f"{sign}£{size:,.2f}" if size < 10 else f"{sign}£{size:,.0f}" + if kind == "count": + if size >= 1e3: + return f"{sign}{size / 1e3:,.1f}k" + return f"{sign}{size:,.0f}" + if kind == "rate": + return f"{sign}{100 * size:.2f}pp" + return f"{sign}{size:.4f}" + + +def _table(header, rows) -> str: + lines = [ + "| " + " | ".join(header) + " |", + "|" + "|".join(["---"] + ["---:"] * (len(header) - 1)) + "|", + ] + lines += ["| " + " | ".join(str(c) for c in row) + " |" for row in rows] + return "\n".join(lines) + + +def _year_header(years): + return [fiscal_year_label(int(y)) for y in years] + + +def _section_rows(summary, section): + return [r for r in summary["rows"] if r["section"] == section] + + +def _changes_table(rows, years) -> str: + changed = [r for r in rows if r["changed"]] + if not changed: + return "No change in any year." + body = [] + for row in changed: + if row["only_in"] == "head": + cells = [_level_cell(row, y, "head") for y in years] + body.append([f"{row['label']} (new: head levels)"] + cells) + elif row["only_in"] == "base": + cells = [_level_cell(row, y, "base") for y in years] + body.append([f"{row['label']} (removed: base levels)"] + cells) + else: + body.append([row["label"]] + [_change_cell(row, y) for y in years]) + return _table(["Measure"] + _year_header(years), body) + + +def _mark(cell) -> str: + # A total that leaves out a suppressed figure that year. + return " †" if cell.get("excludes") else "" + + +def _level_cell(row, year, side) -> str: + cell = row["values"][str(year)] + if cell.get(f"{side}_suppressed"): + return _SUPPRESSED_CELL + return fmt_level(cell[side], row["kind"]) + _mark(cell) + + +def _change_cell(row, year) -> str: + cell = row["values"][str(year)] + if cell.get("delta") is None and ( + cell.get("base_suppressed") or cell.get("head_suppressed") + ): + return _SUPPRESSED_CELL + if cell.get("withheld"): + return _WITHHELD_CELL + _mark(cell) + return fmt_change(cell["delta"], row["kind"]) + _mark(cell) + + +# A figure from fewer than 10 records is not published (privacy.py). +_SUPPRESSED_CELL = "suppressed" +# The sides leave out different suppressed figures, so no change is shown. +_WITHHELD_CELL = "withheld" + + +def _levels_table(rows, years) -> str: + body = [ + [row["label"]] + [_level_cell(row, y, "head") for y in years] for row in rows + ] + return _table(["Measure (head)"] + _year_header(years), body) + + +def _details(summary_text, body) -> str: + return f"
\n{summary_text}\n\n{body}\n\n
" + + +def _deciles(summary, years) -> str: + deciles = summary.get("deciles") or {} + if not deciles: + if summary.get("decile_notes"): + # Every year was withheld or unmatched: say why, not "missing". + return "Not shown:\n\n" + "\n".join( + f"- {note}" for note in summary["decile_notes"] + ) + return "Not available: both sides need household detail." + present = [y for y in years if str(y) in deciles] + if all( + row["mean_change"] == 0 and not row.get("suppressed") + for y in present + for row in deciles[str(y)] + ): + if any(row.get("any_change") for y in present for row in deciles[str(y)]): + return "Every decile's mean change rounds to £0 in every year." + return "No household's net income changes in any year." + labels = [row["decile"] for row in deciles[str(present[0])]] + body = [] + for index, decile in enumerate(labels): + cells = [] + for year in present: + row = deciles[str(year)][index] + if row.get("suppressed"): + cells.append("fewer than 10 households change") + elif row["mean_change"] == 0 or not _finite(row["mean_change"]): + cells.append("0" if row["mean_change"] == 0 else "n/a") + else: + cells.append( + f"{fmt_change(row['mean_change'], 'gbp')} ({100 * row['relative_change']:+.1f}%)" + ) + body.append([decile if decile == "all" else f"Decile {decile}"] + cells) + text = ( + "Mean change in household net income per household, and as a share of the " + "decile's base income; households stay in the base run's deciles.\n\n" + + _table(["Base decile"] + _year_header(present), body) + ) + winners = [] + for year in present: + overall = deciles[str(year)][-1] + if overall.get("suppressed"): + winners.append( + [fiscal_year_label(year)] + ["fewer than 10 households change"] * 2 + ) + continue + winners.append( + [ + fiscal_year_label(year), + f"{100 * overall['share_gaining']:.1f}%", + f"{100 * overall['share_losing']:.1f}%", + ] + ) + text += "\n\n" + _table( + ["Year", "People gaining over £1", "People losing over £1"], winners + ) + if summary.get("decile_notes"): + text += "\n\n" + "\n".join(f"- {note}" for note in summary["decile_notes"]) + return text + + +def _flags(summary) -> str: + flags = list(summary.get("flags") or []) + flags += [ + { + "label": f"Decile {f['decile']} household net income, relative", + "year": f["year"], + "kind": "relative", + "base": None, + "head": None, + "delta": f["relative_change"], + "limit": None, + "section": "deciles", + } + for f in summary.get("decile_flags") or [] + ] + if not flags: + return "None." + flags.sort(key=lambda f: -(abs(f["delta"]) / f["limit"]) if f.get("limit") else 0) + shown = flags[:MAX_FLAG_ROWS] + body = [] + for f in shown: + if f["kind"] == "relative": + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + "", + "", + f"{100 * f['delta']:+.2f}%", + ] + ) + else: + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + fmt_level(f["base"], f["kind"]), + fmt_level(f["head"], f["kind"]), + fmt_change(f["delta"], f["kind"]), + ] + ) + text = _table(["Measure", "Year", "Base", "Head", "Change"], body) + if len(flags) > len(shown): + text += f"\n\n{len(flags) - len(shown)} more flags are in the report artifact." + return text + + +def _check_list(checks, classification) -> list: + out = [] + for check in checks: + if check["classification"] != classification: + continue + details = check["head_details"] or check["base_details"] + detail = f": {'; '.join(details[:3])}" if details else "" + out.append(f"- {check['label']} (`{check['id']}`){detail}") + return out + + +def _guard(summary) -> str: + guard = (summary.get("head") or {}).get("guard") or {} + if guard.get("status") != "ran": + return f"Counted-once check not run: {guard.get('reason', 'unknown reason')}." + components = guard.get("components", []) + tested = [c for c in components if c["status"] != "skipped"] + failed = [c for c in tested if c["status"] == "fail"] + skipped = [c for c in components if c["status"] == "skipped"] + lines = [ + f"Each of {len(tested)} components was raised by £1 per unit in " + f"{fiscal_year_label(guard['year'])}, and every head of its totals was " + "read household by household: its own head must rise by exactly £1, no " + "other tax head may move, no other benefit may rise, and the model's own " + "total must move by what its heads moved. A head that breaks a rule is " + "measured again twice: with every other head held at its baseline value, " + "and with every other head held at its value after the first £1 while the " + "component rises again. It fails where either still shows the response. " + f"{len(failed)} failed; {len(skipped)} were skipped." + ] + for c in failed: + problems = c.get("problems") or [{"message": c.get("detail", "")}] + lines.append( + f"- `{c['variable']}`: " + "; ".join(p["message"] for p in problems) + ) + for c in skipped: + lines.append(f"- `{c['variable']}` skipped: {c.get('detail', '')}") + for c in tested: + for m in c.get("mediated") or []: + moved = ", ".join(f"`{v}`" for v in m["via"]) or "none" + where = ( + f"{m['households']:,} households" + if m.get("households") is not None + else "fewer than 10 households" + ) + lines.append( + f"- `{c['variable']}`: `{m['head']}` responds only through other " + f"heads in {where} (other heads that moved there: {moved}; " + f"{m['total']}; not a failure)" + ) + for dc in guard.get("double_counts", []): + state = "still contained" if dc["status"] == "pass" else "NO LONGER contained" + # The published guard drops the coefficient of a double count from + # fewer than 10 units (measure.publishable_guard). + coefficient = ( + f"coefficient {dc['coefficient']:.3f}" + if _finite(dc.get("coefficient")) + else "coefficient not shown, fewer than 10 units" + ) + lines.append( + f"- declared double count `{dc['variable']}` is {state} in " + f"`{dc['contained_in']}` ({coefficient}; {dc['issue']})" + ) + return "\n".join(lines) + + +def _provenance(summary) -> str: + rows = [] + for side in ("base", "head"): + info = summary.get(side) + if not info: + rows.append([side, "no result", "", "", "", ""]) + continue + provenance = info.get("provenance") or {} + runtime = info.get("runtime") or {} + rows.append( + [ + side, + f"{provenance.get('git_ref', '')} `{str(provenance.get('git_sha', ''))[:10]}`", + f"uk {runtime.get('policyengine_uk')}, core {runtime.get('policyengine_core')}, py {runtime.get('python')} {runtime.get('machine', '')}", + f"`{str(runtime.get('distributions_sha256', ''))[:12]}`", + f"`{str((info.get('partition') or {}).get('sha256', ''))[:12]}`", + f"{(info.get('timings') or {}).get('total', '')}s", + ] + ) + dataset = ((summary.get("head") or {}).get("provenance") or {}).get("dataset") or {} + text = _table( + ["Side", "Commit", "Runtime", "Environment hash", "Partition hash", "Time"], + rows, + ) + if dataset: + text += ( + f"\n\nDataset: `{dataset.get('uri')}`, sha256 `{dataset.get('verified_sha256')}` " + f"(verified on both sides), {dataset.get('data_package')} {dataset.get('data_version')}." + ) + if summary.get("runtime_differences"): + text += "\n\nRuntime differences:\n" + "\n".join( + f"- {d}" for d in summary["runtime_differences"] + ) + return text + + +def render(summary: Mapping, spec: Spec, context: Optional[Mapping] = None) -> str: + """The full comment; sections are dropped from the end to fit GitHub's limit.""" + context = context or {} + if summary["status"] == "skipped": + notes = "\n".join(f"- {w}" for w in summary.get("warnings") or []) + return f"{COMMENT_MARKER}\n## Aggregate impact on certified data: skipped\n\n{notes}\n" + years = summary.get("years") or spec.years + regressions = [c for c in summary["checks"] if c["classification"] == "regression"] + pre_existing = [ + c for c in summary["checks"] if c["classification"] == "pre_existing" + ] + new_warnings = [ + c for c in summary["checks"] if c["classification"] == "new_warning" + ] + fixed = [c for c in summary["checks"] if c["classification"] == "fixed"] + flag_count = len(summary.get("flags") or []) + len( + summary.get("decile_flags") or [] + ) + verdict = "FAIL" if summary["status"] == "fail" else "pass" + + head_prov = (summary.get("head") or {}).get("provenance") or {} + base_prov = (summary.get("base") or {}).get("provenance") or {} + dataset = head_prov.get("dataset") or {} + parts = [ + COMMENT_MARKER, + f"## Aggregate impact on certified data: {verdict}", + "", + ( + f"Base `{str(base_prov.get('git_sha', '?'))[:10]}` against head " + f"`{str(head_prov.get('git_sha', '?'))[:10]}`, both on `{dataset.get('path', '?')}` " + f"({dataset.get('data_package', '')} {dataset.get('data_version', '')}, sha256 " + f"`{str(dataset.get('verified_sha256', '?'))[:12]}`), fiscal years " + f"{fiscal_year_label(years[0])} to {fiscal_year_label(years[-1])}. Each year is a full " + "microsimulation run on each side." + ), + "", + ( + f"**Checks:** {len(regressions)} broken by this change, {len(pre_existing)} failing on " + f"both sides, {len(fixed)} fixed. **Moves flagged for review:** {flag_count} " + "(thresholds in `report_spec.yaml`; a flag does not fail the job)." + ), + ] + if context.get("run_url"): + parts.append(f"\n[Workflow run and full report]({context['run_url']})") + if summary["failures"]: + parts += ["", "### Failures", ""] + [f"- {f}" for f in summary["failures"]] + if new_warnings: + parts += ["", "### New warnings", ""] + _check_list( + summary["checks"], "new_warning" + ) + if summary["warnings"]: + parts += ["", "### Notes", ""] + [f"- {w}" for w in summary["warnings"]] + if fixed: + parts += ["", "### Fixed by this change", ""] + _check_list( + summary["checks"], "fixed" + ) + parts += ["", "### Moves flagged for review", "", _flags(summary)] + + unchanged = [] + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if not rows: + continue + if not any(row["changed"] for row in rows): + unchanged.append(title[:1].lower() + title[1:]) + continue + parts += ["", f"### {title}", "", _changes_table(rows, years)] + parts += ["", "### Household net income by decile", "", _deciles(summary, years)] + if unchanged: + parts += ["", "**No change in any year:** " + "; ".join(unchanged) + "."] + + optional = [] + if pre_existing: + optional.append( + _details( + f"Checks failing on both base and head ({len(pre_existing)})", + "\n".join(_check_list(summary["checks"], "pre_existing")), + ) + ) + optional.append(_details("Counted-once check", _guard(summary))) + optional.append(_details("Provenance", _provenance(summary))) + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if rows: + optional.append( + _details(f"{title}: head levels", _levels_table(rows, years)) + ) + + text = "\n".join(parts) + limit = spec.max_comment_characters + note = "\n\n_Some detail was left out to fit a GitHub comment; the workflow artifact has all of it._" + kept = [] + for block in optional: + candidate = text + "\n\n" + "\n\n".join(kept + [block]) + if len(candidate) + len(note) > limit: + continue + kept.append(block) + if kept: + text += "\n\n" + "\n\n".join(kept) + if len(kept) < len(optional): + text += note + if len(text) > limit: + text = text[: limit - len(note) - 1] + note + return text + "\n" diff --git a/policyengine_uk/utils/aggregate_impact/report_spec.yaml b/policyengine_uk/utils/aggregate_impact/report_spec.yaml new file mode 100644 index 0000000000..a4eb3c6fab --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/report_spec.yaml @@ -0,0 +1,102 @@ +# What the aggregate-impact report measures and when it flags a move. +schema_version: 1 + +# Model periods are fiscal years: 2026 is 2026-27. The horizon spans the +# current fiscal year's predecessor (2025-26, the last year of outturn the +# Budget compares against) through 2031-32, the last year of the Autumn +# Budget 2026 scorecard. Every year is a full microsimulation run. +horizon: + first_year: 2025 + last_year: 2031 + +# Year of the counted-once check, which perturbs every component once. +guard_year: 2026 + +poverty: + measures: + - id: absolute_bhc + label: Absolute poverty, before housing costs + variable: in_poverty_bhc + - id: absolute_ahc + label: Absolute poverty, after housing costs + variable: in_poverty_ahc + - id: relative_bhc + label: Relative poverty, before housing costs + variable: in_relative_poverty_bhc + - id: relative_ahc + label: Relative poverty, after housing costs + variable: in_relative_poverty_ahc + groups: + - id: all + label: Everyone + - id: children + label: Children + variable: is_hbai_dependent_child + - id: working_age_adults + label: Working-age adults + variable: is_hbai_working_age_adult + - id: pensioners + label: Pensioners + variable: is_hbai_pensioner + +inequality: + - id: gini_equiv_hbai_bhc + label: Gini coefficient, equivalised HBAI income before housing costs + variable: equiv_hbai_household_net_income + +deciles: + # Households are placed in deciles by the base run's own decile variable + # (person-weighted equivalised HBAI income); head incomes are compared + # household by household within those base deciles. + decile_variable: household_income_decile + income_variable: household_net_income + +# A move is flagged for review when |head - base| exceeds +# max(absolute, relative * |base|). Flags never fail the job: intended policy +# changes move numbers. +# +# Adopted under cos d1344 (15 October 2026) as a judgement. Flags only decide +# which moves lead the comment: every changed figure is listed in its section +# whether or not it is flagged (in the report artifact, when the comment is +# too long for GitHub and leaves sections out). Repeated runs of the same pair give identical +# figures (the determinism check; W4's two runs of all 1,470 metric rows), so +# no threshold above zero flags noise. £ totals use the larger of £100m and +# 0.1% of the base, so a move must matter for the head it is in and for the +# public finances. Reform costings and reform targets use the larger of +# £100m and 1% of the base costing (reform_gbp), also a judgement: a costing +# leads the comment only when it moves by a material share of itself. Alternatives considered: any non-zero move (flags every +# formula change, so flags stop carrying information) and the replay +# release-diff rule of max(£1m, 1%) (flags £1m moves in small heads but +# nothing under £3bn in income tax). Revisit with the flag counts from the +# first two weeks of pull request comments. +flags: + gbp: + absolute: 100000000 + relative: 0.001 + count: + absolute: 5000 + relative: 0.005 + rate: + absolute: 0.001 + index: + absolute: 0.001 + decile_relative: + absolute: 0.001 + gbp_household: + absolute: 10 + relative: 0.001 + reform_gbp: + absolute: 100000000 + relative: 0.01 + +# Relative tolerance for accounting identities between weighted totals. The +# model stores values as float32, so totals summed in different orders differ +# by a few parts in ten million. +identity_relative_tolerance: 0.000001 +identity_absolute_tolerance_gbp: 1000000 + +# Tolerance on the counted-once coefficient (1 per unit perturbed). +counted_once_tolerance: 0.001 + +comment: + max_characters: 60000 diff --git a/policyengine_uk/utils/aggregate_impact/spec.py b/policyengine_uk/utils/aggregate_impact/spec.py new file mode 100644 index 0000000000..2441253702 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/spec.py @@ -0,0 +1,105 @@ +"""The report specification: horizon, measures and flag thresholds.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Optional + +import yaml + +DEFAULT_SPEC_PATH = Path(__file__).with_name("report_spec.yaml") + + +class SpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Threshold: + absolute: float + relative: float = 0.0 + + def limit(self, base: float) -> float: + return max(self.absolute, self.relative * abs(base)) + + def exceeded(self, base: float, head: float) -> bool: + return abs(head - base) > self.limit(base) + + +@dataclass(frozen=True) +class Spec: + raw: Mapping + first_year: int + last_year: int + guard_year: int + poverty_measures: tuple + poverty_groups: tuple + inequality: tuple + decile_variable: str + income_variable: str + flags: Mapping[str, Threshold] + identity_relative_tolerance: float + identity_absolute_tolerance: float + counted_once_tolerance: float + max_comment_characters: int + + @property + def years(self) -> list: + return list(range(self.first_year, self.last_year + 1)) + + def threshold(self, kind: str) -> Optional[Threshold]: + return self.flags.get(kind) + + +def fiscal_year_label(year: int) -> str: + return f"{year}-{(year + 1) % 100:02d}" + + +def parse_spec(raw: Mapping) -> Spec: + if raw.get("schema_version") != 1: + raise SpecError( + f"unsupported spec schema_version {raw.get('schema_version')!r}" + ) + horizon = raw.get("horizon") or {} + first, last = int(horizon["first_year"]), int(horizon["last_year"]) + if last < first: + raise SpecError("horizon.last_year is before horizon.first_year") + guard_year = int(raw.get("guard_year", first)) + poverty = raw.get("poverty") or {} + flags = { + kind: Threshold(float(v.get("absolute", 0.0)), float(v.get("relative", 0.0))) + for kind, v in (raw.get("flags") or {}).items() + } + deciles = raw.get("deciles") or {} + return Spec( + raw=raw, + first_year=first, + last_year=last, + guard_year=guard_year, + poverty_measures=tuple(poverty.get("measures") or ()), + poverty_groups=tuple(poverty.get("groups") or ()), + inequality=tuple(raw.get("inequality") or ()), + decile_variable=str(deciles.get("decile_variable", "household_income_decile")), + income_variable=str(deciles.get("income_variable", "household_net_income")), + flags=flags, + identity_relative_tolerance=float(raw.get("identity_relative_tolerance", 1e-6)), + identity_absolute_tolerance=float( + raw.get("identity_absolute_tolerance_gbp", 1e6) + ), + counted_once_tolerance=float(raw.get("counted_once_tolerance", 1e-3)), + max_comment_characters=int( + (raw.get("comment") or {}).get("max_characters", 60000) + ), + ) + + +def load_spec(path=None, years: Optional[str] = None) -> Spec: + """Load the spec, optionally overriding the horizon with ``"2025-2031"``.""" + with open(path or DEFAULT_SPEC_PATH) as f: + raw = yaml.safe_load(f) + if years: + start, _, end = years.partition("-") + raw = dict(raw) + raw["horizon"] = {"first_year": int(start), "last_year": int(end or start)} + return parse_spec(raw) diff --git a/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml new file mode 100644 index 0000000000..f4a4467325 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml @@ -0,0 +1,161 @@ +# The standard reforms the aggregate-impact report costs on each side. +# +# Each reform is either absolute (`parameters`: values applied from 2023, as +# the reform-cost fixtures in tests/microsimulation/reforms_config.yaml do) or +# relative (`scale`: each listed parameter is multiplied by the factor in +# every year of the horizon, starting from that year's baseline value). +# +# `responds` states what the reform must do: the model variable whose +# weighted total it must move, the direction, and the least it must move it +# in every year of the horizon. A reform that moves its target by less, by +# nothing or the wrong way fails the `responsive:` check, so a reform the +# model silently ignores cannot pass as a small number. +# +# Calibration (adopted under cos d1344, 15 October 2026). Each minimum is a +# fifth of the reform's smallest yearly effect over 2025-26 to 2031-32 on the +# certified pair, rounded down to two significant figures; parse_reforms +# refuses a minimum that does not follow from its smallest_effect. Recalibrate +# with `python -m policyengine_uk.utils.aggregate_impact calibrate` on fresh +# measurements, and change both numbers together. +# - Pair: uk-data 1.58.1 (enhanced_frs_2024_25, sha256 056e80fd), measured +# with policyengine-uk 2.130.1 on policyengine-core 3.33.0 (the certified +# model, policyengine.py bundle uk-6.4.3) and with 2.132.6 on core 3.33.2. +# Every target change was identical between the two. +# - Rechecked on the same certified data with 2.135.6 on core 3.33.2 +# (merged 7efaaff16, main 62a0117f9, 18 October 2026). The rule changes +# the higher-rate minimum from GBP710m to GBP700m and the additional-rate +# minimum from GBP550m to GBP560m. Their smallest_effect receipts below +# are updated together; the other eight minima are unchanged under the +# rule and retain their original smallest_effect receipts. +# - Rechecked on the same certified data with 2.135.7 on core 3.33.2 +# (merged 7717438bf, main 7ad002f54, 2026-10-18). +# vat_standard_plus_2pp: GBP6100000000 to GBP5000000000. +# Only changed minima update their smallest_effect receipts; all other +# smallest_effect receipts retain their historical values. +# - What the check must catch: a PolicyEngine UK backtest (11 October 2026, +# cited in the Budget CI plan) found the 10% UC standard allowance rise +# returning exactly £0 from 2.45.1 to 2.75.3, and the 10% State Pension +# rise barely moving from 2.88.10 to 2.102.6. +# - What it must not catch: legitimate moves. The fixture receipts show the +# 2029-30 costings moving by up to 18% between data and model versions, +# and 38% for the VAT rise (21.2 on uk-data 1.40.3 against 34.5 on 1.58.0, +# from recorded consumption). A fifth fails a reform only once it has lost +# 80% of its smallest yearly effect, twice the largest move seen. +# - Alternatives considered: a tenth (fails only after a 90% loss, too late +# for "barely moving"); a half (the VAT data change kept 62% of the +# effect, so two such changes in a row would fail a correct model); +# per-year minimums (tighter in years where an absolute reform is worth +# more, but every horizon or data change would need a full table); an +# external anchor such as HMRC's ready reckoner (that tests calibration, +# not whether the reform is wired through); and a sign check alone +# (passes "barely moving"). +# - A single minimum is a wiring check, not a per-year 20% guarantee: the +# personal allowance minimum (£0.35bn) is 20% of its 2031-32 effect +# (£1.78bn) but 8% of its 2030-31 effect (£4.45bn), so a 90% loss in +# 2030-31 alone would still pass. +# +# An absolute reform shrinks as the baseline is uprated towards it: the +# 13,000 personal allowance is worth £4.45bn in 2030-31 but £1.78bn in +# 2031-32, when the baseline allowance rises to 12,830. From 2032-33 the +# baseline (13,090) exceeds it, so before the horizon reaches that year, +# make that reform relative. +schema_version: 1 + +calibration: + fraction: 0.2 + +reforms: + - id: basic_rate_plus_1pp + name: Raise basic rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + responds: + variable: income_tax + direction: up + minimum: 1200000000 + smallest_effect: 6366582049 # 2025-26 + - id: higher_rate_plus_1pp + name: Raise higher rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + responds: + variable: income_tax + direction: up + minimum: 700000000 + smallest_effect: 3544553599 # 2025-26; 2.135.6 recheck above + - id: personal_allowance_13000 + name: Raise personal allowance by ~800GBP/year + parameters: + gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + responds: + variable: income_tax + direction: down + minimum: 350000000 + smallest_effect: 1780300035 # 2031-32 + - id: child_benefit_additional_25 + name: Raise child benefit by 25GBP/week per additional child + parameters: + gov.hmrc.child_benefit.amount.additional: 25 + responds: + variable: child_benefit + direction: up + minimum: 310000000 + smallest_effect: 1591189257 # 2031-32 + - id: uc_taper_20pc + name: Reduce Universal Credit taper rate to 20% + parameters: + gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + responds: + variable: universal_credit + direction: up + minimum: 4100000000 + smallest_effect: 20519585448 # 2025-26 + - id: employee_nics_main_10pc + name: Raise Class 1 main employee NICs rate to 10% + parameters: + gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + responds: + variable: national_insurance + direction: up + minimum: 2200000000 + smallest_effect: 11329957618 # 2025-26 + - id: vat_standard_plus_2pp + name: Raise VAT standard rate by 2pp + parameters: + gov.hmrc.vat.standard_rate: 0.22 + responds: + variable: vat + direction: up + minimum: 5000000000 + smallest_effect: 25006282297 # 2025-26; 2.135.7 recheck above + - id: additional_rate_plus_3pp + name: Raise additional rate by 3pp + parameters: + gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + responds: + variable: income_tax + direction: up + minimum: 560000000 + smallest_effect: 2830314604 # 2025-26; 2.135.6 recheck above + - id: uc_standard_allowance_plus_10pc + name: Raise the Universal Credit standard allowance by 10% + scale: + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_OLD: 1.1 + responds: + variable: universal_credit + direction: up + minimum: 600000000 + smallest_effect: 3041523709 # 2025-26 + - id: state_pension_plus_10pc + name: Raise the State Pension by 10% + scale: + gov.dwp.state_pension.new_state_pension.amount: 1.1 + gov.dwp.state_pension.basic_state_pension.amount: 1.1 + responds: + variable: state_pension + direction: up + minimum: 2400000000 + smallest_effect: 12250304093 # 2025-26