diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1c2ad7fc9e..050239411d 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -52,6 +52,50 @@ and add `verified_end_year` when verified coverage has a defined end. Change the verified year fields only after confirming that parameters and tests cover the stated years. +The registry also defines the budget totals. `household_tax` adds every entry +flagged `budget: tax` and `household_benefits` every entry flagged +`budget: benefit`, in the order the file's `budget_order` lists them. +`policyengine_uk.programs.budget_programs()` returns those terms with their +labels, `tax` or `benefit` and `group`; downstream code reads that function, or +`HOUSEHOLD_TAX_VARIABLES` and `HOUSEHOLD_BENEFIT_VARIABLES`, rather than its own +copy. Four fields set a term's place in the totals: + +- `budget`: `tax` or `benefit` on each entry whose `variable` a total adds; + leave it off the rest. +- `group`: an optional key shared by terms that move together, such as the + childcare entitlements. +- `budget_dropped_when`: the path of a boolean parameter that removes the term + from its total while true (`gov.contrib.abolish_council_tax` on council tax, + the High Value Council Tax Surcharge and Council Tax Reduction). +- `budget_uprating`: which of `gov.contrib.benefit_uprating`'s `all` and + `non_sp` rates scale a benefit term. A benefit term without it takes both. + The State Pension takes `[all]` and basic income `[]`. + +A total's term that no programme entry carries goes under `budget_terms`, with a +`kind`: `programme_measure` (the amount counted for the programme that `program` +names, such as `esa_income` for ESA), `programme` (a programme the coverage list +does not carry yet), `reform_measure` (not a programme: a hypothetical tax or +benefit, or a change from baseline such as `vat_change`, zero unless a reform +sets it) or `public_service` (public services in kind). These entries stay out +of `programs`, so the coverage page never lists them, and they carry no +verified-year fields. The package refuses to load when a budget field is +misspelt or the flags and `budget_order` disagree. + +To add a term to a total: + +1. Flag its entry with `budget`, or add a `budget_terms` entry for it. +2. Append its variable to the end of `budget_order`'s list for that total. +3. Add a head for it to `policyengine_uk/data/fiscal_partition.yaml` that lists + the total in its `totals`: a `tax_heads` entry for a tax term, a + `spending_heads` entry for a benefit term. If the term carries + `budget_dropped_when`, add it under that parameter in the total's + `dropped_when` there too. +4. For a benefit term, also add its variable to `GOV_SPENDING_VARIABLES` in + `policyengine_uk/variables/gov/gov_spending.py`, or, if government does not + pay it, to `NOT_PAID_BY_GOVERNMENT` in + `policyengine_uk/tests/code_health/test_household_benefits_in_government_spending.py` + with the reason. + ## Axiom parity Every policy change here must also be correct in [rulespec-uk](https://github.com/TheAxiomFoundation/rulespec-uk). That covers a new programme, a parameter or threshold update, an eligibility rule and a bug fix. The shared guide, [Mirror policy changes in Axiom](https://github.com/PolicyEngine/.github/blob/main/CONTRIBUTING.md#mirror-policy-changes-in-axiom), defines the `axiom:` line your PR description needs and what a `queued` issue must contain. UK specifics: diff --git a/changelog.d/2702.added.md b/changelog.d/2702.added.md new file mode 100644 index 0000000000..d16b041b95 --- /dev/null +++ b/changelog.d/2702.added.md @@ -0,0 +1 @@ +The budget totals `household_tax` and `household_benefits` now read their terms from the `budget` fields in `programs.yaml`, and `policyengine_uk.programs.budget_programs()` returns each term with its label, tax or benefit flag and group. Calculated values do not change. diff --git a/docs/engineering/skills/model-structure.md b/docs/engineering/skills/model-structure.md index 190b4eea3c..0f345c3f66 100644 --- a/docs/engineering/skills/model-structure.md +++ b/docs/engineering/skills/model-structure.md @@ -63,6 +63,14 @@ Change `verified_start_year` or `verified_end_year` only after verifying that parameters and tests cover the stated years. Current entries use the status values `complete` and `partial`. +The registry also defines the budget totals `household_tax` and +`household_benefits`, and `policyengine_uk.programs.budget_programs()` serves +their terms to downstream code. The `Program registry` section of +`.github/CONTRIBUTING.md` defines the budget fields (`budget`, `group`, +`budget_dropped_when`, `budget_uprating`), the `budget_terms` section and +`budget_order`. Add or remove a term of either total in `programs.yaml`, never +in the formula modules. + ## Refactoring Recovery Lessons During the 2025-05-31 variable refactor recovery, a script that split diff --git a/policyengine_uk/__init__.py b/policyengine_uk/__init__.py index 30f4828b55..20f93ddd69 100644 --- a/policyengine_uk/__init__.py +++ b/policyengine_uk/__init__.py @@ -15,5 +15,6 @@ variables, ) from .model_api import * +from policyengine_uk import programs REPO = Path(__file__).parent diff --git a/policyengine_uk/build_metadata.py b/policyengine_uk/build_metadata.py index 242d4ed6f7..d396e4247b 100644 --- a/policyengine_uk/build_metadata.py +++ b/policyengine_uk/build_metadata.py @@ -50,6 +50,7 @@ "simulation.py", "system.py", "tax_benefit_system.py", + "programs.py", "programs.yaml", ) diff --git a/policyengine_uk/programs.py b/policyengine_uk/programs.py new file mode 100644 index 0000000000..2a58c015b2 --- /dev/null +++ b/policyengine_uk/programs.py @@ -0,0 +1,255 @@ +"""The programme registry and the budget totals built from it. + +``programs.yaml`` lists the programmes the model covers. Its ``budget`` fields +also define the budget totals: ``household_tax`` adds every term flagged +``budget: tax`` and ``household_benefits`` every term flagged +``budget: benefit``, in the order ``budget_order`` gives. PolicyEngine's +simulation API reports the change in ``household_tax`` less the change in +``household_benefits`` as the headline budget impact. + +This module imports only the standard library and ``yaml``, never the +package's top level, so the variable modules can import it while the package +builds the tax-benefit system. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache, reduce +from pathlib import Path +from typing import Optional + +import yaml + +REGISTRY_PATH = Path(__file__).parent / "programs.yaml" + +BUDGET_KINDS = ("tax", "benefit") +# The kind of every `programs` entry, and the kinds a `budget_terms` entry +# may take. +PROGRAMME = "programme" +TERM_KINDS = ("programme_measure", PROGRAMME, "reform_measure", "public_service") +# The gov.contrib.benefit_uprating parameters, in the order +# household_benefits applies them, and the default for a benefit term. +UPRATING_PARAMETERS = ("all", "non_sp") +# The fields that place a term in the totals. The loader refuses any other +# field whose name starts with "budget", so a misspelt one fails loudly. +BUDGET_FIELDS = ("budget", "budget_dropped_when", "budget_uprating") +# Every field a `budget_terms` entry may carry. +TERM_FIELDS = BUDGET_FIELDS + ("variable", "name", "kind", "program", "group", "notes") +# The C parser when PyYAML has it, which parses the registry far faster. +_YAML_LOADER = getattr(yaml, "CSafeLoader", yaml.SafeLoader) + + +class RegistryError(ValueError): + """``programs.yaml`` has malformed or inconsistent budget fields.""" + + +@dataclass(frozen=True) +class BudgetProgram: + """One term of the budget totals. + + Attributes: + variable: the variable the total adds. + label: a display name. + budget: ``"tax"`` for a household_tax term, ``"benefit"`` for a + household_benefits term. + group: a key shared by terms that move together, or None. + kind: ``"programme"``, ``"programme_measure"``, ``"reform_measure"`` + or ``"public_service"`` (see the header of ``programs.yaml``). + Only the first two measure a programme. + program: the id of the ``programs`` entry the term measures or + changes, or None. + dropped_when: path of a boolean parameter that removes the term from + its total while true, or None. + uprating: the gov.contrib.benefit_uprating parameters that scale the + term; empty for taxes. + """ + + variable: str + label: str + budget: str + group: Optional[str] + kind: str + program: Optional[str] + dropped_when: Optional[str] + uprating: tuple + + @property + def is_tax(self) -> bool: + return self.budget == "tax" + + @property + def is_programme(self) -> bool: + return self.kind in (PROGRAMME, "programme_measure") + + +@lru_cache(maxsize=1) +def load_registry() -> dict: + """The parsed ``programs.yaml``.""" + return yaml.load(REGISTRY_PATH.read_text(), Loader=_YAML_LOADER) + + +def _check_budget_fields(entry: dict, name: str) -> None: + unknown = sorted( + key + for key in entry + if str(key).startswith("budget") and key not in BUDGET_FIELDS + ) + if unknown: + raise RegistryError(f"{name}: unknown budget fields {unknown}") + if "budget" not in entry: + stray = sorted(key for key in entry if key in BUDGET_FIELDS) + if stray: + raise RegistryError(f"{name}: {stray} need a budget flag") + + +def _term(entry: dict, kind: str, program: Optional[str]) -> BudgetProgram: + variable = entry["variable"] + budget = entry["budget"] + if budget not in BUDGET_KINDS: + raise RegistryError( + f"{variable}: budget must be tax or benefit, not {budget!r}" + ) + if "budget_uprating" in entry: + if budget != "benefit": + raise RegistryError(f"{variable}: only benefit terms take budget_uprating") + uprating = tuple(entry["budget_uprating"]) + unknown = set(uprating) - set(UPRATING_PARAMETERS) + if unknown: + raise RegistryError( + f"{variable}: unknown budget_uprating {sorted(unknown)}" + ) + else: + uprating = UPRATING_PARAMETERS if budget == "benefit" else () + group = entry.get("group") + if group is not None and not isinstance(group, str): + raise RegistryError(f"{variable}: group must be a string") + dropped_when = entry.get("budget_dropped_when") + if dropped_when is not None and not isinstance(dropped_when, str): + raise RegistryError(f"{variable}: budget_dropped_when must be a parameter path") + return BudgetProgram( + variable=variable, + label=entry["name"], + budget=budget, + group=group, + kind=kind, + program=program, + dropped_when=dropped_when, + uprating=uprating, + ) + + +@lru_cache(maxsize=1) +def budget_programs() -> tuple: + """Every term of the budget totals, as ``BudgetProgram`` records. + + The household_tax terms come first, then the household_benefits terms, + each in the order the total adds them. Every call returns the same + tuple. Raises ``RegistryError`` if a budget field is malformed or + misspelt, or if the registry's budget fields disagree with + ``budget_order``. + """ + return build_budget_programs(load_registry()) + + +def build_budget_programs(registry: dict) -> tuple: + """The budget terms of a parsed registry; see ``budget_programs``.""" + programme_ids = {entry["id"] for entry in registry["programs"]} + terms = {} + + def add(term): + if term.variable in terms: + raise RegistryError(f"{term.variable} carries a budget flag twice") + terms[term.variable] = term + + for entry in registry["programs"]: + _check_budget_fields(entry, entry.get("id", entry.get("variable"))) + if "budget" in entry: + if "variable" not in entry: + raise RegistryError(f"{entry['id']}: a budget entry needs a variable") + add(_term(entry, PROGRAMME, entry["id"])) + for entry in registry.get("budget_terms", []): + name = entry.get("variable") + _check_budget_fields(entry, name) + unknown = sorted(str(key) for key in entry if key not in TERM_FIELDS) + if unknown: + raise RegistryError(f"{name}: unknown budget_terms fields {unknown}") + if "variable" not in entry or "budget" not in entry: + raise RegistryError( + f"{name}: a budget_terms entry needs variable and budget" + ) + kind = entry.get("kind") + if kind not in TERM_KINDS: + raise RegistryError(f"{entry['variable']}: unknown kind {kind!r}") + program = entry.get("program") + if program is not None and program not in programme_ids: + raise RegistryError(f"{entry['variable']}: unknown program {program!r}") + if kind == "programme_measure" and program is None: + raise RegistryError( + f"{entry['variable']}: a programme_measure needs a program" + ) + add(_term(entry, kind, program)) + + ordered = [] + order = registry["budget_order"] + for budget in BUDGET_KINDS: + listed = order[budget] + flagged = [t.variable for t in terms.values() if t.budget == budget] + if len(set(listed)) != len(listed): + raise RegistryError(f"budget_order.{budget} repeats a variable") + if set(listed) != set(flagged): + raise RegistryError( + f"budget_order.{budget} and the entries flagged budget: {budget} " + f"differ: only ordered {sorted(set(listed) - set(flagged))}, " + f"only flagged {sorted(set(flagged) - set(listed))}" + ) + ordered.extend(terms[variable] for variable in listed) + return tuple(ordered) + + +def budget_variables(budget: str) -> list: + """The variables a budget total adds, in order. + + ``budget`` is ``"tax"`` (household_tax) or ``"benefit"`` + (household_benefits). Returns a new list on each call. + """ + if budget not in BUDGET_KINDS: + raise ValueError(f"budget must be tax or benefit, not {budget!r}") + return [term.variable for term in budget_programs() if term.budget == budget] + + +def _parameter_is_true(parameters_at_instant, path: str) -> bool: + return bool(reduce(getattr, path.split("."), parameters_at_instant)) + + +def budget_variables_in_effect(budget: str, parameters_at_instant) -> list: + """The variables a budget total adds under the given parameters. + + Leaves out each term whose ``budget_dropped_when`` parameter is true, + and keeps the order of ``budget_variables``. + """ + return [ + term.variable + for term in budget_programs() + if term.budget == budget + and not ( + term.dropped_when + and _parameter_is_true(parameters_at_instant, term.dropped_when) + ) + ] + + +def uprated_benefit_variables(benefits: list, uprating_parameter: str) -> list: + """The entries of ``benefits`` that ``uprating_parameter`` scales. + + ``uprating_parameter`` is ``"all"`` or ``"non_sp"``, a child of + gov.contrib.benefit_uprating. Keeps the order of ``benefits``. + """ + if uprating_parameter not in UPRATING_PARAMETERS: + raise ValueError(f"unknown uprating parameter {uprating_parameter!r}") + scaled = { + term.variable + for term in budget_programs() + if uprating_parameter in term.uprating + } + return [benefit for benefit in benefits if benefit in scaled] diff --git a/policyengine_uk/programs.yaml b/policyengine_uk/programs.yaml index ba571b4c2e..c0d9083e1a 100644 --- a/policyengine_uk/programs.yaml +++ b/policyengine_uk/programs.yaml @@ -2,6 +2,41 @@ # This is the single source of truth for program coverage information. # Served via the /uk/metadata API as modelled_policies. # Consumed by policyengine-model to render the model coverage page. +# +# The file also defines the budget totals. household_tax and +# household_benefits add the variables flagged with `budget`, in the order +# `budget_order` gives; policyengine_uk.programs reads them and +# budget_programs() returns them with their labels. +# +# Budget fields, on `programs` and `budget_terms` entries: +# budget tax or benefit: household_tax (tax) or +# household_benefits (benefit) adds the entry's +# `variable`. Leave it off entries outside the totals. +# group optional key shared by terms that move together, +# such as the childcare entitlements. +# budget_dropped_when optional path of a boolean parameter; the total +# leaves the term out while the parameter is true. +# budget_uprating benefit terms only: which of +# gov.contrib.benefit_uprating's `all` and `non_sp` +# scale the term. A benefit term without it takes +# both. +# +# `budget_terms` lists the terms that no `programs` entry carries. They stay +# out of the coverage list, and each has a `kind`: +# programme_measure the amount the totals count for the `programs` entry +# that `program` names, or for one part of it +# programme a programme the coverage list does not carry yet; it +# has no coverage or verified-year fields until someone +# confirms them and moves it into `programs` +# reform_measure not a programme: a hypothetical tax or benefit, or a +# change from baseline; it stays zero until a reform +# sets it, and `program` may name the programme it +# changes +# public_service not a programme: public services in kind, at their +# cost to government +# +# The loader refuses a misspelt field whose name starts with `budget`, and +# any `budget_terms` field outside the ones above and `name` and `notes`. programs: # --- HMRC programs --- @@ -25,6 +60,7 @@ programs: status: partial coverage: UK variable: isa_cash_interest_flat_rate_charge + budget: tax parameter_prefix: gov.hmrc.isa.cash_interest_charge verified_start_year: 2027 notes: Separate final manager charge at the savings basic rate from 6 April 2027. Requires the cash-interest component as an input; subscription-limit compliance is not modelled. @@ -48,6 +84,7 @@ programs: status: complete coverage: UK variable: income_tax + budget: tax parameter_prefix: gov.hmrc.income_tax verified_start_year: 2019 notes: Includes Scottish rates via devolved tax bands, the high income child benefit charge, from 2025-26 the winter fuel payment charge, and from 2020-21 the company car and car fuel benefits in kind (no other benefits in kind, and no vans) @@ -60,6 +97,7 @@ programs: status: complete coverage: UK variable: national_insurance + budget: tax parameter_prefix: gov.hmrc.national_insurance verified_start_year: 2019 notes: Classes 1, 2, 3, and 4, and Class 1A on the company car and car fuel benefits @@ -72,6 +110,7 @@ programs: status: complete coverage: UK variable: capital_gains_tax + budget: tax parameter_prefix: gov.hmrc.cgt verified_start_year: 2022 @@ -83,6 +122,7 @@ programs: status: complete coverage: UK variable: child_benefit + budget: benefit parameter_prefix: gov.hmrc.child_benefit verified_start_year: 2019 notes: Includes High Income Child Benefit Charge @@ -117,6 +157,7 @@ programs: status: complete coverage: UK variable: fuel_duty + budget: tax parameter_prefix: gov.hmrc.fuel_duty verified_start_year: 2022 notes: Petrol and diesel, plus supplied kilograms of LPG and natural road fuel gas. Gas consumption is not imputed. @@ -129,6 +170,7 @@ programs: status: partial coverage: UK variable: alcohol_duty + budget: tax parameter_prefix: gov.hmrc.alcohol_duty verified_start_year: 2024 verified_end_year: 2026 @@ -142,6 +184,7 @@ programs: status: partial coverage: UK variable: tobacco_duty + budget: tax parameter_prefix: gov.hmrc.tobacco_duty verified_start_year: 2024 verified_end_year: 2026 @@ -155,6 +198,7 @@ programs: status: partial coverage: UK variable: vehicle_excise_duty + budget: tax parameter_prefix: gov.dft.vehicle_excise_duty verified_start_year: 2024 verified_end_year: 2026 @@ -168,6 +212,7 @@ programs: status: partial coverage: UK variable: gambling_duty + budget: tax parameter_prefix: gov.hmrc.gambling_duty verified_start_year: 2024 verified_end_year: 2027 @@ -192,6 +237,8 @@ programs: status: complete coverage: UK variable: tax_free_childcare + budget: benefit + group: childcare parameter_prefix: gov.hmrc.tax_free_childcare verified_start_year: 2022 @@ -213,6 +260,7 @@ programs: status: partial coverage: UK variable: business_rates + budget: tax parameter_prefix: gov.hmrc.business_rates verified_start_year: 2020 verified_end_year: 2030 @@ -239,6 +287,7 @@ programs: status: complete coverage: UK variable: universal_credit + budget: benefit parameter_prefix: gov.dwp.universal_credit verified_start_year: 2019 notes: Includes standard allowance, child element, housing costs, work allowances, and taper @@ -251,6 +300,7 @@ programs: status: complete coverage: UK variable: pension_credit + budget: benefit parameter_prefix: gov.dwp.pension_credit verified_start_year: 2022 notes: Guarantee Credit and Savings Credit @@ -263,6 +313,8 @@ programs: status: complete coverage: UK variable: state_pension + budget: benefit + budget_uprating: [all] parameter_prefix: gov.dwp.state_pension verified_start_year: 2022 notes: New State Pension and basic State Pension @@ -275,6 +327,7 @@ programs: status: complete coverage: UK variable: housing_benefit + budget: benefit parameter_prefix: gov.dwp.housing_benefit verified_start_year: 2022 notes: Legacy benefit being replaced by Universal Credit @@ -287,6 +340,7 @@ programs: status: complete coverage: UK variable: income_support + budget: benefit parameter_prefix: gov.dwp.income_support verified_start_year: 2022 @@ -322,6 +376,7 @@ programs: status: complete coverage: UK variable: attendance_allowance + budget: benefit parameter_prefix: gov.dwp.attendance_allowance verified_start_year: 2022 @@ -333,6 +388,8 @@ programs: status: complete coverage: UK variable: carers_allowance + budget: benefit + group: carer_benefits parameter_prefix: gov.dwp.carers_allowance verified_start_year: 2022 notes: Reduced by overlapping benefits such as State Pension and contributory ESA @@ -345,6 +402,7 @@ programs: status: complete coverage: UK variable: dla + budget: benefit parameter_prefix: gov.dwp.dla verified_start_year: 2022 @@ -356,6 +414,7 @@ programs: status: complete coverage: UK variable: pip + budget: benefit parameter_prefix: gov.dwp.pip verified_start_year: 2022 @@ -391,6 +450,7 @@ programs: status: complete coverage: UK variable: sda + budget: benefit parameter_prefix: gov.dwp.sda verified_start_year: 2022 @@ -414,6 +474,7 @@ programs: status: complete coverage: UK variable: ssmg + budget: benefit parameter_prefix: gov.dwp.ssmg verified_start_year: 2022 @@ -425,6 +486,7 @@ programs: status: complete coverage: UK variable: maternity_allowance + budget: benefit verified_start_year: 2022 # --- DfE programs --- @@ -436,6 +498,8 @@ programs: status: complete coverage: England variable: universal_childcare_entitlement + budget: benefit + group: childcare parameter_prefix: gov.dfe.universal_childcare_entitlement verified_start_year: 2023 verified_end_year: 2025 @@ -449,6 +513,8 @@ programs: status: complete coverage: England variable: extended_childcare_entitlement + budget: benefit + group: childcare parameter_prefix: gov.dfe.extended_childcare_entitlement verified_start_year: 2023 verified_end_year: 2025 @@ -462,6 +528,8 @@ programs: status: complete coverage: England variable: targeted_childcare_entitlement + budget: benefit + group: childcare parameter_prefix: gov.dfe.targeted_childcare_entitlement verified_start_year: 2023 verified_end_year: 2025 @@ -475,6 +543,8 @@ programs: status: partial coverage: Wales variable: wales_childcare_offer + budget: benefit + group: childcare parameter_prefix: gov.welsh_government.childcare_offer verified_start_year: 2020 verified_end_year: 2026 @@ -488,6 +558,8 @@ programs: status: partial coverage: England variable: childcare_grant + budget: benefit + group: student_support parameter_prefix: gov.dfe.childcare_grant verified_start_year: 2025 notes: First-pass full-time undergraduate England model using explicit full-time, SEN, and childcare-support inputs plus maintenance-loan proxies; eligible children's childcare_expenses are out-of-pocket spend net of free hours, aggregated and capped once per benefit unit. The model allocates the family award to the eldest eligible student parent; actual student partners can share it. @@ -500,6 +572,8 @@ programs: status: partial coverage: England variable: parents_learning_allowance + budget: benefit + group: student_support parameter_prefix: gov.dfe.parents_learning_allowance verified_start_year: 2025 notes: First-pass England model using the published maximum amount and household-income cutoff, with higher-education evidence as the default course proxy and an explicit override for edge cases such as 120-credit or teacher-training routes @@ -513,6 +587,8 @@ programs: status: partial coverage: England variable: adult_dependants_grant + budget: benefit + group: student_support parameter_prefix: gov.dfe.adult_dependants_grant verified_start_year: 2025 notes: First-pass England model using the published maximum amount and household-income tests, with automatic 25-plus couple detection and explicit inputs for under-25 spouse/civil-partner and non-partner adult dependant cases @@ -525,6 +601,8 @@ programs: status: partial coverage: England variable: travel_grant + budget: benefit + group: student_support parameter_prefix: gov.dfe.travel_grant verified_start_year: 2025 notes: First-pass England model using the published initial contribution and household-income reduction formula, with explicit placement and eligible-expense inputs because the survey does not observe reimbursable travel costs @@ -537,6 +615,8 @@ programs: status: partial coverage: England variable: disabled_students_allowance + budget: benefit + group: student_support parameter_prefix: gov.dfe.disabled_students_allowance verified_start_year: 2025 notes: First-pass England model using the published annual cap, higher-education evidence as the default course proxy, a narrow qualifying-condition proxy with explicit override, and explicit disability-related study support costs @@ -549,6 +629,8 @@ programs: status: partial coverage: England variable: bursary_fund_16_to_19 + budget: benefit + group: student_support parameter_prefix: gov.dfe.bursary_fund_16_to_19 verified_start_year: 2025 notes: First-pass model of the defined vulnerable-group bursary only, using explicit care, self-supporting, and participation-cost inputs plus person-level reported-benefit proxies @@ -562,6 +644,7 @@ programs: status: complete coverage: UK variable: tv_licence + budget: tax parameter_prefix: gov.dcms.bbc verified_start_year: 2022 @@ -574,6 +657,9 @@ programs: status: complete coverage: Great Britain variable: council_tax + budget: tax + group: council_tax + budget_dropped_when: gov.contrib.abolish_council_tax verified_start_year: 2022 - id: council_tax_reduction @@ -609,6 +695,7 @@ programs: status: partial coverage: UK variable: energy_bills_rebate + budget: benefit parameter_prefix: gov.treasury.energy_bills_rebate verified_start_year: 2022 verified_end_year: 2023 @@ -629,6 +716,7 @@ programs: status: complete coverage: England variable: household_energy_payment + budget: benefit parameter_prefix: gov.treasury.household_energy_payment verified_start_year: 2026 verified_end_year: 2026 @@ -693,6 +781,7 @@ programs: status: complete coverage: Scotland variable: scottish_child_payment + budget: benefit parameter_prefix: gov.social_security_scotland.scottish_child_payment verified_start_year: 2022 @@ -707,3 +796,418 @@ programs: parameter_prefix: gov.social_security_scotland.pawhp verified_start_year: 2024 notes: Replaced the Winter Fuel Payment in Scotland from winter 2024-25. From winter 2025-26 recovered by the winter fuel payment charge (income tax) above £35,000 of total income + +# Budget terms that no programme entry above carries. The coverage list +# (`programs`) leaves these out: some measure part of a listed programme, +# some a programme the list doesn't carry yet, some no programme. `kind` +# says which; see the header. +budget_terms: + # --- Taxes --- + - variable: expected_sdlt + name: Stamp Duty Land Tax on household purchases + budget: tax + kind: programme_measure + program: stamp_duty_land_tax + group: property_transaction_taxes + + - variable: expected_ltt + name: Land Transaction Tax on household purchases + budget: tax + kind: programme + group: property_transaction_taxes + notes: Welsh Revenue Authority's tax on property purchases in Wales. Not yet in the coverage list. + + - variable: expected_lbtt + name: Land and Buildings Transaction Tax on household purchases + budget: tax + kind: programme_measure + program: lbtt + group: property_transaction_taxes + + - variable: corporate_sdlt + name: Stamp Duty Land Tax paid by companies + budget: tax + kind: programme_measure + program: stamp_duty_land_tax + group: property_transaction_taxes + notes: Stamp Duty Land Tax that companies pay; the model shares it across households by shareholding. + + - variable: high_value_council_tax_surcharge + name: High Value Council Tax Surcharge + budget: tax + kind: programme + group: council_tax + budget_dropped_when: gov.contrib.abolish_council_tax + notes: Not yet in the coverage list. + + - variable: domestic_rates + name: Domestic rates + budget: tax + kind: programme + notes: Northern Ireland's domestic rates. Not yet in the coverage list. + + - variable: wealth_tax + name: Wealth tax + budget: tax + kind: reform_measure + notes: A hypothetical tax, zero unless a reform sets its rates. + + - variable: non_primary_residence_wealth_tax + name: Wealth tax excluding the main residence + budget: tax + kind: reform_measure + notes: A hypothetical tax, zero unless a reform sets its rates. + + - variable: LVT + name: Land value tax + budget: tax + kind: reform_measure + notes: A hypothetical tax, zero unless a reform sets its rates. + + - variable: carbon_tax + name: Carbon tax + budget: tax + kind: reform_measure + notes: A hypothetical tax, zero unless a reform sets its rates. + + - variable: vat_change + name: Change in VAT + budget: tax + kind: reform_measure + program: vat + group: vat + notes: VAT less baseline VAT. The totals count this change, not the VAT level (`vat`). + + - variable: private_school_vat_change + name: Change in private school VAT + budget: tax + kind: reform_measure + program: vat + group: vat + notes: Private school VAT less its baseline. + + - variable: corporate_incident_tax_revenue_change + name: Unspecified tax change borne by capital + budget: tax + kind: reform_measure + notes: A lump-sum revenue change that a reform sets; the model shares it across households by shareholding. + + - variable: consumer_incident_tax_revenue_change + name: Unspecified tax change borne by consumers + budget: tax + kind: reform_measure + notes: A lump-sum revenue change that a reform sets; the model shares it across households by consumption. + + - variable: employer_ni_response_capital_incidence + name: Employer National Insurance change borne by capital + budget: tax + kind: reform_measure + group: employer_ni_incidence + notes: The share of a change in employer National Insurance that falls on capital. The totals count this change, not employer National Insurance (`ni_employer`). + + - variable: employer_ni_response_consumer_incidence + name: Employer National Insurance change borne by consumers + budget: tax + kind: reform_measure + group: employer_ni_incidence + notes: The share of a change in employer National Insurance that falls on consumers through prices. + + - variable: student_loan_repayments + name: Student loan repayments + budget: tax + kind: programme_measure + program: student_loans + + # --- Benefits --- + - variable: council_tax_benefit + name: Council Tax Reduction + budget: benefit + kind: programme_measure + program: council_tax_reduction + budget_dropped_when: gov.contrib.abolish_council_tax + + - variable: esa_income + name: ESA (income-related) + budget: benefit + kind: programme_measure + program: esa + group: esa + + - variable: esa_contrib + name: ESA (contribution-based) + budget: benefit + kind: programme_measure + program: esa + group: esa + + - variable: jsa_income + name: JSA (income-based) + budget: benefit + kind: programme_measure + program: jsa + group: jsa + + - variable: jsa_contrib + name: JSA (contribution-based) + budget: benefit + kind: programme_measure + program: jsa + group: jsa + + - variable: working_tax_credit + name: Working Tax Credit + budget: benefit + kind: programme_measure + program: tax_credits + group: tax_credits + + - variable: child_tax_credit + name: Child Tax Credit + budget: benefit + kind: programme_measure + program: tax_credits + group: tax_credits + + - variable: afcs + name: Armed Forces Compensation Scheme + budget: benefit + kind: programme + notes: Reported amounts. Not yet in the coverage list. + + - variable: bsp + name: Bereavement Support Payment + budget: benefit + kind: programme + notes: Not yet in the coverage list. + + - variable: war_widows_pension + name: War Widow's or Widower's Pension + budget: benefit + kind: programme + notes: Reported amounts. Not yet in the coverage list. + + - variable: widowed_parents_allowance + name: Widowed Parent's Allowance + budget: benefit + kind: programme + notes: Not yet in the coverage list. + + - variable: iidb + name: Industrial Injuries Disablement Benefit + budget: benefit + kind: programme + notes: Reported amounts. Not yet in the coverage list. + + - variable: incapacity_benefit + name: Incapacity Benefit + budget: benefit + kind: programme + notes: Not yet in the coverage list. + + - variable: statutory_sick_pay + name: Statutory Sick Pay + budget: benefit + kind: programme + group: statutory_pay + notes: Not yet in the coverage list. + + - variable: statutory_maternity_pay + name: Statutory Maternity Pay + budget: benefit + kind: programme + group: statutory_pay + notes: Not yet in the coverage list. + + - variable: statutory_paternity_pay + name: Statutory Paternity Pay + budget: benefit + kind: programme + group: statutory_pay + notes: Not yet in the coverage list. + + - variable: basic_income + name: Basic income + budget: benefit + kind: reform_measure + budget_uprating: [] + notes: A hypothetical benefit, zero unless a reform sets its amounts. + + - variable: epg_subsidy + name: Energy Price Guarantee subsidy + budget: benefit + kind: programme_measure + program: energy_price_guarantee + + - variable: cost_of_living_support_payment + name: Cost of Living Support payments + budget: benefit + kind: programme_measure + program: cost_of_living_support + + - variable: winter_fuel_allowance + name: Winter Fuel Payment + budget: benefit + kind: programme_measure + program: winter_fuel_payment + group: winter_heating_payments + + - variable: pawhp + name: Pension Age Winter Heating Payment + budget: benefit + kind: programme_measure + program: pawhp + group: winter_heating_payments + + - variable: care_to_learn + name: Care to Learn + budget: benefit + kind: programme + group: student_support + notes: Not yet in the coverage list. + + - variable: nhs_spending + name: NHS spending + budget: benefit + kind: public_service + group: public_services_in_kind + notes: Public services the household uses, at their cost to government. Not a cash programme. + + - variable: dfe_education_spending + name: State education spending + budget: benefit + kind: public_service + group: public_services_in_kind + notes: Public services the household uses, at their cost to government. Not a cash programme. + + - variable: dft_subsidy_spending + name: Transport subsidy spending + budget: benefit + kind: public_service + group: public_services_in_kind + notes: Public services the household uses, at their cost to government. Not a cash programme. + + - variable: healthy_start_vouchers + name: Healthy Start vouchers + budget: benefit + kind: programme + notes: Not yet in the coverage list. + + - variable: two_child_limit_payment + name: Two Child Limit Payment + budget: benefit + kind: reform_measure + notes: A proposed Scottish payment, whose regulations the Scottish Government confirmed on 3 December 2025 it will not lay. Zero unless a reform switches it on. + + - variable: carer_support_payment + name: Carer Support Payment + budget: benefit + kind: programme + group: carer_benefits + notes: Scotland's replacement for Carer's Allowance. Not yet in the coverage list. + + - variable: scottish_carer_supplement + name: Carer's Allowance Supplement + budget: benefit + kind: programme + group: carer_benefits + notes: Scotland's supplement to Carer's Allowance. Not yet in the coverage list. + + - variable: disability_basic_income + name: Disability basic income + budget: benefit + kind: reform_measure + notes: A hypothetical benefit, zero unless a reform sets its amount. + +# The order in which household_tax and household_benefits add their terms. +# Each list holds every variable whose entry above has that `budget` value, +# once. Float sums depend on order, so append a new term at the end of its +# list rather than reordering. +budget_order: + tax: + - isa_cash_interest_flat_rate_charge + - expected_sdlt + - expected_ltt + - expected_lbtt + - corporate_sdlt + - business_rates + - council_tax + - high_value_council_tax_surcharge + - domestic_rates + - fuel_duty + - alcohol_duty + - tobacco_duty + - gambling_duty + - vehicle_excise_duty + - tv_licence + - wealth_tax + - non_primary_residence_wealth_tax + - income_tax + - national_insurance + - LVT + - carbon_tax + - vat_change + - capital_gains_tax + - private_school_vat_change + - corporate_incident_tax_revenue_change + - consumer_incident_tax_revenue_change + - employer_ni_response_capital_incidence + - employer_ni_response_consumer_incidence + - student_loan_repayments + benefit: + - child_benefit + - council_tax_benefit + - esa_income + - esa_contrib + - housing_benefit + - income_support + - jsa_income + - jsa_contrib + - pension_credit + - universal_credit + - working_tax_credit + - child_tax_credit + - attendance_allowance + - afcs + - bsp + - war_widows_pension + - widowed_parents_allowance + - carers_allowance + - dla + - iidb + - incapacity_benefit + - pip + - sda + - state_pension + - maternity_allowance + - statutory_sick_pay + - statutory_maternity_pay + - statutory_paternity_pay + - ssmg + - basic_income + - epg_subsidy + - cost_of_living_support_payment + - energy_bills_rebate + - household_energy_payment + - winter_fuel_allowance + - pawhp + - tax_free_childcare + - extended_childcare_entitlement + - universal_childcare_entitlement + - targeted_childcare_entitlement + - wales_childcare_offer + - care_to_learn + - childcare_grant + - parents_learning_allowance + - adult_dependants_grant + - travel_grant + - disabled_students_allowance + - bursary_fund_16_to_19 + - nhs_spending + - dfe_education_spending + - dft_subsidy_spending + - healthy_start_vouchers + - two_child_limit_payment + - scottish_child_payment + - carer_support_payment + - scottish_carer_supplement + - disability_basic_income diff --git a/policyengine_uk/tests/test_budget_programs.py b/policyengine_uk/tests/test_budget_programs.py new file mode 100644 index 0000000000..6391829bc3 --- /dev/null +++ b/policyengine_uk/tests/test_budget_programs.py @@ -0,0 +1,627 @@ +"""The budget totals add the budget terms in programs.yaml and no others. + +household_tax adds the programs.yaml terms flagged ``budget: tax`` and +household_benefits those flagged ``budget: benefit``, in ``budget_order``. +The drift tests parse the YAML themselves rather than through +policyengine_uk.programs. The tests check: + +1. Drift: each total's formula reads the flagged terms and no others, less + those whose ``budget_dropped_when`` parameter is true, in the order its + ``add`` calls imply; the fiscal partition's ``dropped_when`` for each + total names the same terms as the registry. +2. Differential: HOUSEHOLD_TAX_VARIABLES and HOUSEHOLD_BENEFIT_VARIABLES + equal, in order, the lists the formula modules held as literals at commit + 2da212abcf2086337fc088a4a1236f4ae7b6af63, and the council tax and + uprating rules match the ones those formulas applied. +3. budget_programs() returns the same records in a fresh interpreter, + whichever module that interpreter imports first. +4. Properties: every budget_programs() variable exists, none repeats, and + each sign agrees with the list it came from; budget_variables_in_effect + and uprated_benefit_variables return subsequences of budget_order, in + order; on generated registries (Hypothesis), the loader keeps + budget_order's order and refuses a registry whose flags and order + disagree. +5. Validation: the loader refuses each kind of malformed or misspelt budget + field. +""" + +import copy +import json +import re +import subprocess +import sys +from types import SimpleNamespace + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem, Simulation +from policyengine_uk.programs import ( + REGISTRY_PATH, + BudgetProgram, + RegistryError, + build_budget_programs, + budget_programs, + budget_variables, + budget_variables_in_effect, + uprated_benefit_variables, +) +from policyengine_uk.utils.aggregate_impact.partition import ( + default_partition_path, + load_partition, +) +from policyengine_uk.variables.gov.hmrc.household_tax import HOUSEHOLD_TAX_VARIABLES +from policyengine_uk.variables.household.income.household_benefits import ( + HOUSEHOLD_BENEFIT_VARIABLES, +) + +SYSTEM = CountryTaxBenefitSystem() +RAW = yaml.safe_load(REGISTRY_PATH.read_text()) +ABOLISH_COUNCIL_TAX = "gov.contrib.abolish_council_tax" +LISTS = {"tax": HOUSEHOLD_TAX_VARIABLES, "benefit": HOUSEHOLD_BENEFIT_VARIABLES} +TOTALS = {"tax": "household_tax", "benefit": "household_benefits"} + +# The lists as household_tax.py and household_benefits.py held them at +# commit 2da212ab, before the totals read programs.yaml. +MAIN_HOUSEHOLD_TAX_VARIABLES = [ + "isa_cash_interest_flat_rate_charge", + "expected_sdlt", + "expected_ltt", + "expected_lbtt", + "corporate_sdlt", + "business_rates", + "council_tax", + "high_value_council_tax_surcharge", + "domestic_rates", + "fuel_duty", + "alcohol_duty", + "tobacco_duty", + "gambling_duty", + "vehicle_excise_duty", + "tv_licence", + "wealth_tax", + "non_primary_residence_wealth_tax", + "income_tax", + "national_insurance", + "LVT", + "carbon_tax", + "vat_change", + "capital_gains_tax", + "private_school_vat_change", + "corporate_incident_tax_revenue_change", + "consumer_incident_tax_revenue_change", + "employer_ni_response_capital_incidence", + "employer_ni_response_consumer_incidence", + "student_loan_repayments", +] + +MAIN_HOUSEHOLD_BENEFIT_VARIABLES = [ + "child_benefit", + "council_tax_benefit", + "esa_income", + "esa_contrib", + "housing_benefit", + "income_support", + "jsa_income", + "jsa_contrib", + "pension_credit", + "universal_credit", + "working_tax_credit", + "child_tax_credit", + "attendance_allowance", + "afcs", + "bsp", + "war_widows_pension", + "widowed_parents_allowance", + "carers_allowance", + "dla", + "iidb", + "incapacity_benefit", + "pip", + "sda", + "state_pension", + "maternity_allowance", + "statutory_sick_pay", + "statutory_maternity_pay", + "statutory_paternity_pay", + "ssmg", + "basic_income", + "epg_subsidy", + "cost_of_living_support_payment", + "energy_bills_rebate", + "household_energy_payment", + "winter_fuel_allowance", + "pawhp", + "tax_free_childcare", + "extended_childcare_entitlement", + "universal_childcare_entitlement", + "targeted_childcare_entitlement", + "wales_childcare_offer", + "care_to_learn", + "childcare_grant", + "parents_learning_allowance", + "adult_dependants_grant", + "travel_grant", + "disabled_students_allowance", + "bursary_fund_16_to_19", + "nhs_spending", + "dfe_education_spending", + "dft_subsidy_spending", + "healthy_start_vouchers", + "two_child_limit_payment", + "scottish_child_payment", + "carer_support_payment", + "scottish_carer_supplement", + "disability_basic_income", +] + +# The rules those formulas applied: abolishing council tax removed these +# terms, the non_sp uprating left out the State Pension, and neither uprating +# scaled basic income. +MAIN_DROPPED_WHEN_ABOLISHING_COUNCIL_TAX = { + "council_tax", + "high_value_council_tax_surcharge", + "council_tax_benefit", +} +MAIN_UPRATING_EXCEPTIONS = {"state_pension": ("all",), "basic_income": ()} + + +def flagged(budget): + """Variables the raw YAML flags with ``budget``, from both sections.""" + entries = RAW["programs"] + RAW["budget_terms"] + return [entry["variable"] for entry in entries if entry.get("budget") == budget] + + +# 1. Drift + + +@pytest.mark.parametrize("budget", ["tax", "benefit"]) +def test_registry_flags_are_the_listed_terms(budget): + names = flagged(budget) + assert len(names) == len(set(names)) + assert sorted(names) == sorted(LISTS[budget]) + assert RAW["budget_order"][budget] == LISTS[budget] + + +def read_by_formula(total, reform): + """The variables ``total``'s formula reads, in the order it reads them.""" + year = 2026 + sim = Simulation( + situation={ + "people": {"adult": {"age": {year: 40}}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": {"household": {"members": ["adult"]}}, + }, + reform=reform or None, + ) + sim.trace = True + sim.calculate(total, year) + (root,) = [tree for tree in sim.tracer.trees if tree.name == total] + return [child.name for child in root.children] + + +@pytest.mark.parametrize("abolish", [False, True], ids=["current_law", "abolished"]) +@pytest.mark.parametrize("budget", ["tax", "benefit"]) +def test_totals_read_exactly_the_registry_terms(budget, abolish): + reform = {ABOLISH_COUNCIL_TAX: True} if abolish else {} + expected = [ + name + for name in RAW["budget_order"][budget] + if not (abolish and name in MAIN_DROPPED_WHEN_ABOLISHING_COUNCIL_TAX) + ] + read = read_by_formula(TOTALS[budget], reform) + assert set(read) == set(expected) + if budget == "tax": + # household_tax makes one add() call, so the trace lists its reads in + # summation order. + assert read == expected + else: + # household_benefits adds the terms the `all` rate scales, then those + # the `non_sp` rate scales, then every term; the trace lists the + # first read of each term in that order. + assert list(dict.fromkeys(read)) == list( + dict.fromkeys( + raw_uprated(expected, "all") + + raw_uprated(expected, "non_sp") + + expected + ) + ) + + +def raw_uprated(benefits, rate): + """The entries of ``benefits`` the raw YAML lets ``rate`` scale, in order.""" + uprating = { + entry["variable"]: entry.get("budget_uprating", ["all", "non_sp"]) + for entry in RAW["programs"] + RAW["budget_terms"] + if entry.get("budget") == "benefit" + } + return [benefit for benefit in benefits if rate in uprating[benefit]] + + +@pytest.mark.parametrize("budget", ["tax", "benefit"]) +def test_partition_drops_the_registry_terms(budget): + """fiscal_partition.yaml's dropped_when for each total names the terms + that carry the same budget_dropped_when in programs.yaml.""" + partition = load_partition(default_partition_path()) + entries = [ + entry + for entry in RAW["programs"] + RAW["budget_terms"] + if entry.get("budget") == budget + ] + switches = { + entry["budget_dropped_when"] + for entry in entries + if "budget_dropped_when" in entry + } | set(partition.dropped_when_parameters()) + assert switches + for switch in switches: + assert partition.dropped(TOTALS[budget], [switch]) == { + entry["variable"] + for entry in entries + if entry.get("budget_dropped_when") == switch + }, switch + + +# 2. Differential against main + + +def test_household_tax_variables_equal_main(): + assert HOUSEHOLD_TAX_VARIABLES == MAIN_HOUSEHOLD_TAX_VARIABLES + assert budget_variables("tax") == MAIN_HOUSEHOLD_TAX_VARIABLES + + +def test_household_benefit_variables_equal_main(): + assert HOUSEHOLD_BENEFIT_VARIABLES == MAIN_HOUSEHOLD_BENEFIT_VARIABLES + assert budget_variables("benefit") == MAIN_HOUSEHOLD_BENEFIT_VARIABLES + + +def test_council_tax_and_uprating_rules_equal_main(): + dropped = {p.variable for p in budget_programs() if p.dropped_when} + assert dropped == MAIN_DROPPED_WHEN_ABOLISHING_COUNCIL_TAX + assert {p.dropped_when for p in budget_programs() if p.dropped_when} == { + ABOLISH_COUNCIL_TAX + } + for program in budget_programs(): + if program.is_tax: + assert program.uprating == () + else: + expected = MAIN_UPRATING_EXCEPTIONS.get(program.variable, ("all", "non_sp")) + assert program.uprating == expected, program.variable + + +# 3. Fresh interpreter + +FRESH_SCRIPT = """ +import dataclasses, json, sys +if sys.argv[1] == "programs": + from policyengine_uk.programs import budget_programs +elif sys.argv[1] == "household_tax": + from policyengine_uk.variables.gov.hmrc.household_tax import HOUSEHOLD_TAX_VARIABLES + from policyengine_uk.programs import budget_programs +else: + import policyengine_uk + budget_programs = policyengine_uk.programs.budget_programs +print(json.dumps([dataclasses.asdict(p) for p in budget_programs()])) +""" + + +@pytest.mark.parametrize("first_import", ["programs", "household_tax", "package"]) +def test_budget_programs_in_a_fresh_interpreter(first_import): + result = subprocess.run( + [sys.executable, "-c", FRESH_SCRIPT, first_import], + capture_output=True, + text=True, + check=True, + ) + fresh = json.loads(result.stdout.strip().splitlines()[-1]) + here = [ + {**program.__dict__, "uprating": list(program.uprating)} + for program in budget_programs() + ] + assert fresh == here + + +def test_budget_programs_have_labels_and_kinds(): + for program in budget_programs(): + assert isinstance(program, BudgetProgram) + assert program.label.strip() + assert program.kind in { + "programme", + "programme_measure", + "reform_measure", + "public_service", + } + if program.kind == "programme_measure": + assert program.program is not None + ids = {entry["id"] for entry in RAW["programs"]} + non_programmes = [p for p in budget_programs() if not p.is_programme] + assert {p.variable for p in non_programmes} >= { + "vat_change", + "employer_ni_response_capital_incidence", + "employer_ni_response_consumer_incidence", + } + # A term that is not a programme never sits in the coverage list. + coverage_variables = {entry.get("variable") for entry in RAW["programs"]} + assert not {p.variable for p in non_programmes} & coverage_variables + assert all(p.program in ids for p in budget_programs() if p.program) + + +# 4. Properties + +PROGRAMS = budget_programs() + + +@pytest.mark.parametrize("program", PROGRAMS, ids=lambda p: p.variable) +def test_each_budget_program_exists_once_with_its_list_sign(program): + assert program.variable in SYSTEM.variables + assert [p.variable for p in PROGRAMS].count(program.variable) == 1 + assert program.budget in ("tax", "benefit") + own, other = ( + (HOUSEHOLD_TAX_VARIABLES, HOUSEHOLD_BENEFIT_VARIABLES) + if program.is_tax + else (HOUSEHOLD_BENEFIT_VARIABLES, HOUSEHOLD_TAX_VARIABLES) + ) + assert program.variable in own + assert program.variable not in other + + +def is_ordered_subsequence(part, whole): + position = iter(whole) + return all(item in position for item in part) + + +def parameters_with(switches): + """A stand-in for parameters at an instant, with each switch path set.""" + root = SimpleNamespace() + for path, value in switches.items(): + node = root + *parents, leaf = path.split(".") + for name in parents: + if not hasattr(node, name): + setattr(node, name, SimpleNamespace()) + node = getattr(node, name) + setattr(node, leaf, value) + return root + + +SWITCHES = sorted({p.dropped_when for p in PROGRAMS if p.dropped_when}) + + +@settings(max_examples=100, deadline=None, derandomize=True) +@given( + st.sampled_from(["tax", "benefit"]), + st.fixed_dictionaries({switch: st.booleans() for switch in SWITCHES}), + st.sampled_from(["all", "non_sp"]), + st.data(), +) +def test_selections_are_ordered_subsequences_of_budget_order( + budget, switches, rate, data +): + order = RAW["budget_order"][budget] + in_effect = budget_variables_in_effect(budget, parameters_with(switches)) + assert is_ordered_subsequence(in_effect, order) + on = {path for path, value in switches.items() if value} + assert in_effect == [ + p.variable for p in PROGRAMS if p.budget == budget and p.dropped_when not in on + ] + benefits = RAW["budget_order"]["benefit"] + chosen = data.draw(st.lists(st.sampled_from(benefits), unique=True)) + subset = [benefit for benefit in benefits if benefit in chosen] + for selection in ( + subset, + budget_variables_in_effect("benefit", parameters_with(switches)), + ): + uprated = uprated_benefit_variables(selection, rate) + assert is_ordered_subsequence(uprated, selection) + assert is_ordered_subsequence(uprated, benefits) + + +names = st.text(alphabet="abcdefghij_", min_size=1, max_size=8) + + +@st.composite +def registries(draw): + """A registry with flagged terms in both sections and a shuffled order.""" + variables = draw(st.lists(names, min_size=1, max_size=12, unique=True)) + budgets = [draw(st.sampled_from(["tax", "benefit"])) for _ in variables] + in_programs = [draw(st.booleans()) for _ in variables] + programs, terms = [], [] + for variable, budget, listed in zip(variables, budgets, in_programs): + entry = {"variable": variable, "name": variable.title(), "budget": budget} + if listed: + programs.append({"id": f"id_{variable}", **entry}) + else: + terms.append({**entry, "kind": "reform_measure"}) + order = { + budget: draw( + st.permutations([v for v, b in zip(variables, budgets) if b == budget]) + ) + for budget in ("tax", "benefit") + } + return {"programs": programs, "budget_terms": terms, "budget_order": order} + + +@settings(max_examples=100, deadline=None, derandomize=True) +@given(registries()) +def test_loader_keeps_budget_order_and_signs(registry): + built = build_budget_programs(registry) + for budget in ("tax", "benefit"): + assert [p.variable for p in built if p.budget == budget] == list( + registry["budget_order"][budget] + ) + variables = [p.variable for p in built] + assert len(variables) == len(set(variables)) + # Taxes come before benefits. + kinds = [p.budget for p in built] + assert kinds == sorted(kinds, key=["tax", "benefit"].index) + + +@settings(max_examples=100, deadline=None, derandomize=True) +@given(registries(), st.data()) +def test_loader_refuses_order_that_disagrees_with_flags(registry, data): + budget = data.draw( + st.sampled_from([b for b in ("tax", "benefit") if registry["budget_order"][b]]) + ) + order = list(registry["budget_order"][budget]) + if data.draw(st.booleans()): + order.pop(data.draw(st.integers(0, len(order) - 1))) + else: + order.append(data.draw(st.sampled_from(order))) + broken = {**registry, "budget_order": {**registry["budget_order"], budget: order}} + with pytest.raises(RegistryError): + build_budget_programs(broken) + + +# 5. Validation + +VALID = { + "programs": [ + { + "id": "council_tax", + "name": "Council Tax", + "variable": "council_tax", + "budget": "tax", + "budget_dropped_when": "gov.contrib.abolish_council_tax", + }, + {"id": "esa", "name": "ESA", "variable": "esa"}, + ], + "budget_terms": [ + { + "variable": "basic_income", + "name": "Basic income", + "budget": "benefit", + "kind": "reform_measure", + "budget_uprating": [], + }, + { + "variable": "esa_income", + "name": "ESA (income-related)", + "budget": "benefit", + "kind": "programme_measure", + "program": "esa", + "group": "esa", + }, + ], + "budget_order": { + "tax": ["council_tax"], + "benefit": ["basic_income", "esa_income"], + }, +} + + +def test_valid_example_registry_loads(): + built = build_budget_programs(VALID) + assert [p.variable for p in built] == ["council_tax", "basic_income", "esa_income"] + assert built[0].dropped_when == "gov.contrib.abolish_council_tax" + assert built[1].uprating == () + assert built[2].uprating == ("all", "non_sp") + + +def _set(path, value): + def change(registry): + section, index, key = path + registry[section][index][key] = value + + return change + + +def _delete(path): + def change(registry): + section, index, key = path + del registry[section][index][key] + + return change + + +def _unflag_term_with_dropped_when(registry): + entry = registry["budget_terms"][1] + del entry["budget"] + entry["budget_dropped_when"] = "gov.contrib.abolish_council_tax" + + +# One malformed registry per check the loader makes, with the message it +# gives. The checks on budget field names run on both sections, so each +# section has its own case. +INVALID = { + "budget_not_tax_or_benefit": ( + _set(("programs", 0, "budget"), "spending"), + "budget must be tax or benefit", + ), + "uprating_on_a_tax": ( + _set(("programs", 0, "budget_uprating"), ["all"]), + "only benefit terms take budget_uprating", + ), + "unknown_uprating": ( + _set(("budget_terms", 0, "budget_uprating"), ["sp"]), + "unknown budget_uprating", + ), + "group_not_a_string": ( + _set(("budget_terms", 1, "group"), ["esa"]), + "group must be a string", + ), + "dropped_when_not_a_path": ( + _set(("programs", 0, "budget_dropped_when"), True), + "budget_dropped_when must be a parameter path", + ), + "flagged_programme_without_variable": ( + _delete(("programs", 0, "variable")), + "a budget entry needs a variable", + ), + "unknown_kind": (_set(("budget_terms", 0, "kind"), "measure"), "unknown kind"), + "unknown_program": ( + _set(("budget_terms", 1, "program"), "esa_typo"), + "unknown program", + ), + "programme_measure_without_program": ( + _delete(("budget_terms", 1, "program")), + "a programme_measure needs a program", + ), + "budget_term_without_budget": ( + _delete(("budget_terms", 1, "budget")), + "needs variable and budget", + ), + "misspelt_dropped_when": ( + _set(("programs", 0, "budget_droped_when"), "gov.contrib.abolish_council_tax"), + "unknown budget fields ['budget_droped_when']", + ), + "misspelt_uprating": ( + _set(("budget_terms", 0, "budget_upratng"), []), + "unknown budget fields ['budget_upratng']", + ), + "unknown_budget_terms_field": ( + _set(("budget_terms", 1, "dropped_when"), "gov.contrib.abolish_council_tax"), + "unknown budget_terms fields ['dropped_when']", + ), + "budget_field_without_flag": ( + _set(("programs", 1, "budget_uprating"), ["all"]), + "need a budget flag", + ), + "budget_term_field_without_flag": ( + _unflag_term_with_dropped_when, + "need a budget flag", + ), + "budget_term_without_variable": ( + _delete(("budget_terms", 1, "variable")), + "needs variable and budget", + ), + "flagged_twice": ( + lambda r: r["programs"][1].update(variable="esa_income", budget="benefit"), + "carries a budget flag twice", + ), + "order_repeats": ( + lambda r: r["budget_order"]["tax"].append("council_tax"), + "repeats a variable", + ), + "order_disagrees": (lambda r: r["budget_order"]["benefit"].pop(), "differ"), +} + + +@pytest.mark.parametrize("case", list(INVALID)) +def test_loader_refuses_malformed_registry(case): + change, message = INVALID[case] + registry = copy.deepcopy(VALID) + change(registry) + with pytest.raises(RegistryError, match=re.escape(message)): + build_budget_programs(registry) diff --git a/policyengine_uk/tests/test_budget_totals_pinned.py b/policyengine_uk/tests/test_budget_totals_pinned.py new file mode 100644 index 0000000000..2a2d5286e4 --- /dev/null +++ b/policyengine_uk/tests/test_budget_totals_pinned.py @@ -0,0 +1,1083 @@ +"""household_tax, household_benefits and household_net_income stay exact. + +household_tax and household_benefits read their terms from programs.yaml. +EXPECTED holds the three totals for thirteen hand-built households in four +years under four reforms, computed with the lists that the formula modules +held as literals at commit 46a60678ff52d829253572393abdcc6a03ceb6a7. The +reforms switch on gov.contrib.abolish_council_tax, the two +gov.contrib.benefit_uprating rates, and a flat basic income, which the +uprating leaves out. The test compares with exact equality, because the +totals add the same float32 terms in the same order. +""" + +import pytest + +from policyengine_uk import Simulation + +YEARS = (2023, 2025, 2026, 2028) + +REFORMS = { + "baseline": {}, + "abolish_council_tax": {"gov.contrib.abolish_council_tax": True}, + "uprating": { + "gov.contrib.benefit_uprating.all": 0.05, + "gov.contrib.benefit_uprating.non_sp": 0.1, + }, + "abolish_uprating_basic_income": { + "gov.contrib.abolish_council_tax": True, + "gov.contrib.benefit_uprating.all": -0.03, + "gov.contrib.benefit_uprating.non_sp": 0.07, + "gov.contrib.ubi_center.basic_income.amount.flat": 40, + }, +} + +# Each household: region, tenure, rent, council tax, main residence value, +# and its people. +HOUSEHOLDS = { + "single_renter_london": dict( + region="LONDON", + tenure_type="RENT_PRIVATELY", + rent=14_400, + council_tax=1_500, + people={"a": dict(age=30, employment_income=25_000)}, + ), + "couple_two_children_uc": dict( + region="NORTH_WEST", + tenure_type="RENT_FROM_COUNCIL", + rent=6_000, + council_tax=1_400, + would_claim_uc=True, + people={ + "a": dict(age=34, employment_income=18_000), + "b": dict(age=32), + "c1": dict(age=3), + "c2": dict(age=7), + }, + ), + "pensioner_couple": dict( + region="SOUTH_WEST", + tenure_type="OWNED_OUTRIGHT", + council_tax=2_100, + people={ + "a": dict(age=72, state_pension_reported=11_000), + "b": dict(age=69, state_pension_reported=9_000), + }, + ), + "scottish_lone_parent": dict( + region="SCOTLAND", + tenure_type="RENT_FROM_HA", + rent=5_200, + council_tax=1_100, + would_claim_uc=True, + people={ + "a": dict(age=26, employment_income=9_000), + "c1": dict(age=2), + }, + ), + "welsh_working_family": dict( + region="WALES", + tenure_type="OWNED_WITH_MORTGAGE", + council_tax=1_800, + people={ + "a": dict(age=38, employment_income=42_000), + "b": dict(age=36, employment_income=21_000), + "c1": dict(age=3), + }, + ), + "northern_ireland_owner": dict( + region="NORTHERN_IRELAND", + tenure_type="OWNED_OUTRIGHT", + people={ + "a": dict(age=50, self_employment_income=38_000), + "b": dict(age=48, employment_income=15_000), + }, + ), + "high_earner_south_east": dict( + region="SOUTH_EAST", + tenure_type="OWNED_WITH_MORTGAGE", + council_tax=3_200, + people={ + "a": dict(age=45, employment_income=160_000), + "c1": dict(age=10), + "c2": dict(age=13), + }, + ), + "unemployed_contributory_jsa": dict( + region="WEST_MIDLANDS", + tenure_type="RENT_PRIVATELY", + rent=7_800, + council_tax=1_300, + people={"a": dict(age=27, jsa_contrib_reported=3_000)}, + ), + "esa_and_reported_benefits": dict( + region="NORTH_EAST", + tenure_type="RENT_FROM_COUNCIL", + rent=5_000, + council_tax=1_200, + people={ + "a": dict(age=55, esa_contrib_reported=5_000, iidb_reported=2_000), + "b": dict(age=52, employment_income=12_000, afcs_reported=1_500), + }, + ), + "scottish_pensioner": dict( + region="SCOTLAND", + tenure_type="OWNED_OUTRIGHT", + council_tax=1_600, + people={"a": dict(age=81, state_pension_reported=12_500)}, + ), + "large_family_yorkshire": dict( + region="YORKSHIRE", + tenure_type="RENT_PRIVATELY", + rent=9_600, + council_tax=1_500, + would_claim_uc=True, + people={ + "a": dict(age=40, employment_income=24_000), + "b": dict(age=39), + "c1": dict(age=1), + "c2": dict(age=4), + "c3": dict(age=9), + "c4": dict(age=12), + }, + ), + "east_of_england_mixed": dict( + region="EAST_OF_ENGLAND", + tenure_type="OWNED_WITH_MORTGAGE", + council_tax=2_000, + people={ + "a": dict(age=63, employment_income=30_000, self_employment_income=8_000), + "b": dict(age=67, state_pension_reported=10_500, bsp_reported=1_000), + }, + ), + # Owns a £3m main residence, so pays the High Value Council Tax + # Surcharge from 2028. + "london_high_value_owner": dict( + region="LONDON", + tenure_type="OWNED_OUTRIGHT", + council_tax=3_800, + main_residence_value=3_000_000, + people={"a": dict(age=58, employment_income=180_000)}, + ), +} + +HOUSEHOLD_INPUTS = ( + "region", + "tenure_type", + "rent", + "council_tax", + "main_residence_value", +) +OUTPUTS = ("household_tax", "household_benefits", "household_net_income") + + +def situation(year): + """All the households above, as one situation for ``year``.""" + people, benunits, households = {}, {}, {} + for key, spec in HOUSEHOLDS.items(): + members = [] + for person, inputs in spec["people"].items(): + name = f"{key}.{person}" + people[name] = {var: {year: value} for var, value in inputs.items()} + members.append(name) + benunits[key] = { + "members": members, + "would_claim_uc": {year: spec.get("would_claim_uc", False)}, + } + households[key] = {"members": members} + for var in HOUSEHOLD_INPUTS: + if var in spec: + households[key][var] = {year: spec[var]} + return {"people": people, "benunits": benunits, "households": households} + + +# (reform, year, household): (household_tax, household_benefits, +# household_net_income), from commit 46a60678. +EXPECTED = { + ("baseline", 2023, "single_renter_london"): ( + 5636.60498046875, + 201.0, + 19564.39453125, + ), + ("baseline", 2023, "couple_two_children_uc"): ( + 3044.604736328125, + 18789.482421875, + 33744.87890625, + ), + ("baseline", 2023, "pensioner_couple"): (2259.0, 22177.880859375, 19918.880859375), + ("baseline", 2023, "scottish_lone_parent"): ( + 1259.0, + 14060.240234375, + 21801.240234375, + ), + ("baseline", 2023, "welsh_working_family"): ( + 14074.208984375, + 6699.0, + 55624.7890625, + ), + ("baseline", 2023, "northern_ireland_owner"): (8490.705078125, 600.0, 45109.296875), + ("baseline", 2023, "high_earner_south_east"): ( + 71726.2265625, + 2275.800048828125, + 90549.5703125, + ), + ("baseline", 2023, "unemployed_contributory_jsa"): (1459.0, 3201.0, 1742.0), + ("baseline", 2023, "esa_and_reported_benefits"): (1359.0, 8701.0, 19342.0), + ("baseline", 2023, "scottish_pensioner"): ( + 1600.0, + 15566.208984375, + 13966.208984375, + ), + ("baseline", 2023, "large_family_yorkshire"): ( + 5064.60498046875, + 19482.12109375, + 38417.515625, + ), + ("baseline", 2023, "east_of_england_mixed"): ( + 9084.6044921875, + 12201.0, + 41116.39453125, + ), + ("baseline", 2023, "london_high_value_owner"): ( + 79902.2265625, + 201.0, + 100298.7734375, + ), + ("baseline", 2025, "single_renter_london"): (5154.9033203125, 0.0, 19845.09765625), + ("baseline", 2025, "couple_two_children_uc"): ( + 2842.903076171875, + 18652.060546875, + 33809.16015625, + ), + ("baseline", 2025, "pensioner_couple"): (2274.5, 22106.400390625, 19831.900390625), + ("baseline", 2025, "scottish_lone_parent"): ( + 1274.5, + 14044.3994140625, + 21769.8984375, + ), + ("baseline", 2025, "welsh_working_family"): ( + 12575.306640625, + 8074.599609375, + 58499.296875, + ), + ("baseline", 2025, "northern_ireland_owner"): (7466.703125, 0.0, 45533.296875), + ("baseline", 2025, "high_earner_south_east"): ( + 69039.1015625, + 2251.599609375, + 93212.4921875, + ), + ("baseline", 2025, "unemployed_contributory_jsa"): (1474.5, 3000.0, 1525.5), + ("baseline", 2025, "esa_and_reported_benefits"): (1374.5, 8500.0, 19125.5), + ("baseline", 2025, "scottish_pensioner"): ( + 1600.0, + 14945.4013671875, + 13345.4013671875, + ), + ("baseline", 2025, "large_family_yorkshire"): ( + 4622.9033203125, + 21061.541015625, + 40438.63671875, + ), + ("baseline", 2025, "east_of_england_mixed"): ( + 8402.9033203125, + 11700.0, + 41297.09765625, + ), + ("baseline", 2025, "london_high_value_owner"): (76788.1015625, 0.0, 103211.8984375), + ("baseline", 2026, "single_renter_london"): ( + 5160.4033203125, + 100.0, + 19939.59765625, + ), + ("baseline", 2026, "couple_two_children_uc"): ( + 2848.403076171875, + 19863.220703125, + 35014.81640625, + ), + ("baseline", 2026, "pensioner_couple"): (2280.0, 22386.83984375, 20106.83984375), + ("baseline", 2026, "scottish_lone_parent"): (1280.0, 14687.2802734375, 22407.28125), + ("baseline", 2026, "welsh_working_family"): ( + 12580.806640625, + 8410.099609375, + 58829.296875, + ), + ("baseline", 2026, "northern_ireland_owner"): (7472.203125, 0.0, 45527.796875), + ("baseline", 2026, "high_earner_south_east"): ( + 69130.6015625, + 2437.39990234375, + 93306.8046875, + ), + ("baseline", 2026, "unemployed_contributory_jsa"): (1480.0, 3100.0, 1620.0), + ("baseline", 2026, "esa_and_reported_benefits"): (1380.0, 8600.0, 19220.0), + ("baseline", 2026, "scottish_pensioner"): (1600.0, 15286.47265625, 13686.47265625), + ("baseline", 2026, "large_family_yorkshire"): ( + 4628.4033203125, + 29562.759765625, + 48934.35546875, + ), + ("baseline", 2026, "east_of_england_mixed"): ( + 8408.4033203125, + 11800.0, + 41391.59765625, + ), + ("baseline", 2026, "london_high_value_owner"): ( + 76793.6015625, + 100.0, + 103306.3984375, + ), + ("baseline", 2028, "single_renter_london"): (5160.4033203125, 0.0, 19839.59765625), + ("baseline", 2028, "couple_two_children_uc"): ( + 2848.403076171875, + 21507.8984375, + 36659.49609375, + ), + ("baseline", 2028, "pensioner_couple"): (2280.0, 22573.318359375, 20293.318359375), + ("baseline", 2028, "scottish_lone_parent"): ( + 1280.0, + 15792.3837890625, + 23512.3828125, + ), + ("baseline", 2028, "welsh_working_family"): ( + 12580.806640625, + 8973.591796875, + 59392.7890625, + ), + ("baseline", 2028, "northern_ireland_owner"): (7472.203125, 0.0, 45527.796875), + ("baseline", 2028, "high_earner_south_east"): ( + 69294.6015625, + 2501.4619140625, + 93206.8671875, + ), + ("baseline", 2028, "unemployed_contributory_jsa"): (1480.0, 3000.0, 1520.0), + ("baseline", 2028, "esa_and_reported_benefits"): (1380.0, 8500.0, 19120.0), + ("baseline", 2028, "scottish_pensioner"): ( + 1600.0, + 15751.6220703125, + 14151.6220703125, + ), + ("baseline", 2028, "large_family_yorkshire"): ( + 4628.4033203125, + 31807.14453125, + 51178.7421875, + ), + ("baseline", 2028, "east_of_england_mixed"): ( + 8408.4033203125, + 11700.0, + 41291.59765625, + ), + ("baseline", 2028, "london_high_value_owner"): (80293.6015625, 0.0, 99706.3984375), + ("abolish_council_tax", 2023, "single_renter_london"): ( + 4136.60498046875, + 201.0, + 21064.39453125, + ), + ("abolish_council_tax", 2023, "couple_two_children_uc"): ( + 1644.604736328125, + 18789.482421875, + 35144.87890625, + ), + ("abolish_council_tax", 2023, "pensioner_couple"): (159.0, 20701.0, 20542.0), + ("abolish_council_tax", 2023, "scottish_lone_parent"): ( + 159.0, + 14060.240234375, + 22901.240234375, + ), + ("abolish_council_tax", 2023, "welsh_working_family"): ( + 12274.208984375, + 6699.0, + 57424.7890625, + ), + ("abolish_council_tax", 2023, "northern_ireland_owner"): ( + 8490.705078125, + 600.0, + 45109.296875, + ), + ("abolish_council_tax", 2023, "high_earner_south_east"): ( + 68526.2265625, + 2275.800048828125, + 93749.5703125, + ), + ("abolish_council_tax", 2023, "unemployed_contributory_jsa"): ( + 159.0, + 3201.0, + 3042.0, + ), + ("abolish_council_tax", 2023, "esa_and_reported_benefits"): ( + 159.0, + 8701.0, + 20542.0, + ), + ("abolish_council_tax", 2023, "scottish_pensioner"): ( + 0.0, + 14211.51171875, + 14211.51171875, + ), + ("abolish_council_tax", 2023, "large_family_yorkshire"): ( + 3564.60498046875, + 19482.12109375, + 39917.515625, + ), + ("abolish_council_tax", 2023, "east_of_england_mixed"): ( + 7084.6044921875, + 12201.0, + 43116.39453125, + ), + ("abolish_council_tax", 2023, "london_high_value_owner"): ( + 76102.2265625, + 201.0, + 104098.7734375, + ), + ("abolish_council_tax", 2025, "single_renter_london"): ( + 3654.903076171875, + 0.0, + 21345.09765625, + ), + ("abolish_council_tax", 2025, "couple_two_children_uc"): ( + 1442.903076171875, + 18652.060546875, + 35209.16015625, + ), + ("abolish_council_tax", 2025, "pensioner_couple"): (174.5, 20200.0, 20025.5), + ("abolish_council_tax", 2025, "scottish_lone_parent"): ( + 174.5, + 14044.3994140625, + 22869.8984375, + ), + ("abolish_council_tax", 2025, "welsh_working_family"): ( + 10775.306640625, + 8074.599609375, + 60299.296875, + ), + ("abolish_council_tax", 2025, "northern_ireland_owner"): ( + 7466.703125, + 0.0, + 45533.296875, + ), + ("abolish_council_tax", 2025, "high_earner_south_east"): ( + 65839.1015625, + 2251.599609375, + 96412.4921875, + ), + ("abolish_council_tax", 2025, "unemployed_contributory_jsa"): ( + 174.5, + 3000.0, + 2825.5, + ), + ("abolish_council_tax", 2025, "esa_and_reported_benefits"): ( + 174.5, + 8500.0, + 20325.5, + ), + ("abolish_council_tax", 2025, "scottish_pensioner"): ( + 0.0, + 13428.2763671875, + 13428.2763671875, + ), + ("abolish_council_tax", 2025, "large_family_yorkshire"): ( + 3122.903076171875, + 21061.541015625, + 41938.63671875, + ), + ("abolish_council_tax", 2025, "east_of_england_mixed"): ( + 6402.9033203125, + 11700.0, + 43297.09765625, + ), + ("abolish_council_tax", 2025, "london_high_value_owner"): ( + 72988.1015625, + 0.0, + 107011.8984375, + ), + ("abolish_council_tax", 2026, "single_renter_london"): ( + 3660.403076171875, + 100.0, + 21439.59765625, + ), + ("abolish_council_tax", 2026, "couple_two_children_uc"): ( + 1448.403076171875, + 19863.220703125, + 36414.81640625, + ), + ("abolish_council_tax", 2026, "pensioner_couple"): (180.0, 20300.0, 20120.0), + ("abolish_council_tax", 2026, "scottish_lone_parent"): ( + 180.0, + 14687.2802734375, + 23507.28125, + ), + ("abolish_council_tax", 2026, "welsh_working_family"): ( + 10780.806640625, + 8410.099609375, + 60629.296875, + ), + ("abolish_council_tax", 2026, "northern_ireland_owner"): ( + 7472.203125, + 0.0, + 45527.796875, + ), + ("abolish_council_tax", 2026, "high_earner_south_east"): ( + 65930.6015625, + 2437.39990234375, + 96506.8046875, + ), + ("abolish_council_tax", 2026, "unemployed_contributory_jsa"): ( + 180.0, + 3100.0, + 2920.0, + ), + ("abolish_council_tax", 2026, "esa_and_reported_benefits"): ( + 180.0, + 8600.0, + 20420.0, + ), + ("abolish_council_tax", 2026, "scottish_pensioner"): ( + 0.0, + 13700.916015625, + 13700.916015625, + ), + ("abolish_council_tax", 2026, "large_family_yorkshire"): ( + 3128.403076171875, + 29562.759765625, + 50434.35546875, + ), + ("abolish_council_tax", 2026, "east_of_england_mixed"): ( + 6408.4033203125, + 11800.0, + 43391.59765625, + ), + ("abolish_council_tax", 2026, "london_high_value_owner"): ( + 72993.6015625, + 100.0, + 107106.3984375, + ), + ("abolish_council_tax", 2028, "single_renter_london"): ( + 3660.403076171875, + 0.0, + 21339.59765625, + ), + ("abolish_council_tax", 2028, "couple_two_children_uc"): ( + 1448.403076171875, + 21507.8984375, + 38059.49609375, + ), + ("abolish_council_tax", 2028, "pensioner_couple"): ( + 180.0, + 20473.318359375, + 20293.318359375, + ), + ("abolish_council_tax", 2028, "scottish_lone_parent"): ( + 180.0, + 15792.3837890625, + 24612.3828125, + ), + ("abolish_council_tax", 2028, "welsh_working_family"): ( + 10780.806640625, + 8973.591796875, + 61192.7890625, + ), + ("abolish_council_tax", 2028, "northern_ireland_owner"): ( + 7472.203125, + 0.0, + 45527.796875, + ), + ("abolish_council_tax", 2028, "high_earner_south_east"): ( + 66094.6015625, + 2501.4619140625, + 96406.8671875, + ), + ("abolish_council_tax", 2028, "unemployed_contributory_jsa"): ( + 180.0, + 3000.0, + 2820.0, + ), + ("abolish_council_tax", 2028, "esa_and_reported_benefits"): ( + 180.0, + 8500.0, + 20320.0, + ), + ("abolish_council_tax", 2028, "scottish_pensioner"): ( + 0.0, + 14151.6220703125, + 14151.6220703125, + ), + ("abolish_council_tax", 2028, "large_family_yorkshire"): ( + 3128.403076171875, + 31807.14453125, + 52678.7421875, + ), + ("abolish_council_tax", 2028, "east_of_england_mixed"): ( + 6408.4033203125, + 11700.0, + 43291.59765625, + ), + ("abolish_council_tax", 2028, "london_high_value_owner"): ( + 72993.6015625, + 0.0, + 107006.3984375, + ), + ("uprating", 2023, "single_renter_london"): ( + 5636.60498046875, + 231.14999389648438, + 19594.544921875, + ), + ("uprating", 2023, "couple_two_children_uc"): ( + 3044.604736328125, + 21607.90625, + 36563.30078125, + ), + ("uprating", 2023, "pensioner_couple"): (2259.0, 23504.5625, 21245.5625), + ("uprating", 2023, "scottish_lone_parent"): ( + 1259.0, + 16169.2763671875, + 23910.27734375, + ), + ("uprating", 2023, "welsh_working_family"): ( + 14074.208984375, + 7703.85009765625, + 56629.640625, + ), + ("uprating", 2023, "northern_ireland_owner"): (8490.705078125, 690.0, 45199.296875), + ("uprating", 2023, "high_earner_south_east"): ( + 71726.2265625, + 2617.170166015625, + 90890.9453125, + ), + ("uprating", 2023, "unemployed_contributory_jsa"): ( + 1459.0, + 3681.14990234375, + 2222.14990234375, + ), + ("uprating", 2023, "esa_and_reported_benefits"): ( + 1359.0, + 10006.150390625, + 20647.150390625, + ), + ("uprating", 2023, "scottish_pensioner"): (1600.0, 16651.140625, 15051.140625), + ("uprating", 2023, "large_family_yorkshire"): ( + 5064.60498046875, + 22404.439453125, + 41339.83203125, + ), + ("uprating", 2023, "east_of_england_mixed"): ( + 9084.6044921875, + 12981.150390625, + 41896.54296875, + ), + ("uprating", 2023, "london_high_value_owner"): ( + 79902.2265625, + 231.14999389648438, + 100328.9296875, + ), + ("uprating", 2025, "single_renter_london"): (5154.9033203125, 0.0, 19845.09765625), + ("uprating", 2025, "couple_two_children_uc"): ( + 2842.903076171875, + 21449.87109375, + 36606.96875, + ), + ("uprating", 2025, "pensioner_couple"): (2274.5, 23422.359375, 21147.859375), + ("uprating", 2025, "scottish_lone_parent"): ( + 1274.5, + 16151.0595703125, + 23876.55859375, + ), + ("uprating", 2025, "welsh_working_family"): ( + 12575.306640625, + 9285.7900390625, + 59710.484375, + ), + ("uprating", 2025, "northern_ireland_owner"): (7466.703125, 0.0, 45533.296875), + ("uprating", 2025, "high_earner_south_east"): ( + 69039.1015625, + 2589.339599609375, + 93550.2421875, + ), + ("uprating", 2025, "unemployed_contributory_jsa"): (1474.5, 3450.0, 1975.5), + ("uprating", 2025, "esa_and_reported_benefits"): (1374.5, 9775.0, 20400.5), + ("uprating", 2025, "scottish_pensioner"): (1600.0, 15937.2109375, 14337.2109375), + ("uprating", 2025, "large_family_yorkshire"): ( + 4622.9033203125, + 24220.771484375, + 43597.87109375, + ), + ("uprating", 2025, "east_of_england_mixed"): ( + 8402.9033203125, + 12405.0, + 42002.09765625, + ), + ("uprating", 2025, "london_high_value_owner"): (76788.1015625, 0.0, 103211.8984375), + ("uprating", 2026, "single_renter_london"): ( + 5160.4033203125, + 115.0, + 19954.59765625, + ), + ("uprating", 2026, "couple_two_children_uc"): ( + 2848.403076171875, + 22842.703125, + 37994.30078125, + ), + ("uprating", 2026, "pensioner_couple"): (2280.0, 23744.865234375, 21464.865234375), + ("uprating", 2026, "scottish_lone_parent"): ( + 1280.0, + 16890.373046875, + 24610.373046875, + ), + ("uprating", 2026, "welsh_working_family"): ( + 12580.806640625, + 9671.615234375, + 60090.8125, + ), + ("uprating", 2026, "northern_ireland_owner"): (7472.203125, 0.0, 45527.796875), + ("uprating", 2026, "high_earner_south_east"): ( + 69130.6015625, + 2803.009765625, + 93672.4140625, + ), + ("uprating", 2026, "unemployed_contributory_jsa"): (1480.0, 3565.0, 2085.0), + ("uprating", 2026, "esa_and_reported_benefits"): (1380.0, 9890.0, 20510.0), + ("uprating", 2026, "scottish_pensioner"): ( + 1600.0, + 16329.443359375, + 14729.443359375, + ), + ("uprating", 2026, "large_family_yorkshire"): ( + 4628.4033203125, + 33997.171875, + 53368.76953125, + ), + ("uprating", 2026, "east_of_england_mixed"): ( + 8408.4033203125, + 12520.0, + 42111.59765625, + ), + ("uprating", 2026, "london_high_value_owner"): ( + 76793.6015625, + 115.0, + 103321.3984375, + ), + ("uprating", 2028, "single_renter_london"): (5160.4033203125, 0.0, 19839.59765625), + ("uprating", 2028, "couple_two_children_uc"): ( + 2848.403076171875, + 24734.08203125, + 39885.6796875, + ), + ("uprating", 2028, "pensioner_couple"): (2280.0, 23959.31640625, 21679.31640625), + ("uprating", 2028, "scottish_lone_parent"): ( + 1280.0, + 18161.240234375, + 25881.240234375, + ), + ("uprating", 2028, "welsh_working_family"): ( + 12580.806640625, + 10319.630859375, + 60738.828125, + ), + ("uprating", 2028, "northern_ireland_owner"): (7472.203125, 0.0, 45527.796875), + ("uprating", 2028, "high_earner_south_east"): ( + 69294.6015625, + 2876.68115234375, + 93582.0859375, + ), + ("uprating", 2028, "unemployed_contributory_jsa"): (1480.0, 3450.0, 1970.0), + ("uprating", 2028, "esa_and_reported_benefits"): (1380.0, 9775.0, 20395.0), + ("uprating", 2028, "scottish_pensioner"): ( + 1600.0, + 16864.365234375, + 15264.365234375, + ), + ("uprating", 2028, "large_family_yorkshire"): ( + 4628.4033203125, + 36578.21484375, + 55949.8125, + ), + ("uprating", 2028, "east_of_england_mixed"): ( + 8408.4033203125, + 12405.0, + 41996.59765625, + ), + ("uprating", 2028, "london_high_value_owner"): (80293.6015625, 0.0, 99706.3984375), + ("abolish_uprating_basic_income", 2023, "single_renter_london"): ( + 4136.60498046875, + 2289.0400390625, + 23152.43359375, + ), + ("abolish_uprating_basic_income", 2023, "couple_two_children_uc"): ( + 1644.604736328125, + 27861.0625, + 44216.45703125, + ), + ("abolish_uprating_basic_income", 2023, "pensioner_couple"): ( + 159.0, + 24289.0390625, + 24130.0390625, + ), + ("abolish_uprating_basic_income", 2023, "scottish_lone_parent"): ( + 159.0, + 18782.650390625, + 27623.650390625, + ), + ("abolish_uprating_basic_income", 2023, "welsh_working_family"): ( + 12274.208984375, + 13206.9599609375, + 63932.75, + ), + ("abolish_uprating_basic_income", 2023, "northern_ireland_owner"): ( + 8490.705078125, + 4784.0, + 49293.296875, + ), + ("abolish_uprating_basic_income", 2023, "high_earner_south_east"): ( + 68526.2265625, + 8606.83203125, + 100080.6015625, + ), + ("abolish_uprating_basic_income", 2023, "unemployed_contributory_jsa"): ( + 159.0, + 5409.0400390625, + 5250.0400390625, + ), + ("abolish_uprating_basic_income", 2023, "esa_and_reported_benefits"): ( + 159.0, + 13209.0400390625, + 25050.0390625, + ), + ("abolish_uprating_basic_income", 2023, "scottish_pensioner"): ( + 0.0, + 15984.9716796875, + 15984.9716796875, + ), + ("abolish_uprating_basic_income", 2023, "large_family_yorkshire"): ( + 3564.60498046875, + 32741.40625, + 53176.80078125, + ), + ("abolish_uprating_basic_income", 2023, "east_of_england_mixed"): ( + 7084.6044921875, + 16114.0400390625, + 47029.43359375, + ), + ("abolish_uprating_basic_income", 2023, "london_high_value_owner"): ( + 76102.2265625, + 2289.0400390625, + 106186.8203125, + ), + ("abolish_uprating_basic_income", 2025, "single_renter_london"): ( + 3654.903076171875, + 2080.0, + 23425.09765625, + ), + ("abolish_uprating_basic_income", 2025, "couple_two_children_uc"): ( + 1442.903076171875, + 27718.142578125, + 44275.23828125, + ), + ("abolish_uprating_basic_income", 2025, "pensioner_couple"): ( + 174.5, + 23768.0, + 23593.5, + ), + ("abolish_uprating_basic_income", 2025, "scottish_lone_parent"): ( + 174.5, + 18766.17578125, + 27591.67578125, + ), + ("abolish_uprating_basic_income", 2025, "welsh_working_family"): ( + 10775.306640625, + 14637.583984375, + 66862.28125, + ), + ("abolish_uprating_basic_income", 2025, "northern_ireland_owner"): ( + 7466.703125, + 4160.0, + 49693.296875, + ), + ("abolish_uprating_basic_income", 2025, "high_earner_south_east"): ( + 65839.1015625, + 8581.6640625, + 102742.5546875, + ), + ("abolish_uprating_basic_income", 2025, "unemployed_contributory_jsa"): ( + 174.5, + 5200.0, + 5025.5, + ), + ("abolish_uprating_basic_income", 2025, "esa_and_reported_benefits"): ( + 174.5, + 13000.0, + 24825.5, + ), + ("abolish_uprating_basic_income", 2025, "scottish_pensioner"): ( + 0.0, + 15170.4072265625, + 15170.4072265625, + ), + ("abolish_uprating_basic_income", 2025, "large_family_yorkshire"): ( + 3122.903076171875, + 34384.0, + 55261.09765625, + ), + ("abolish_uprating_basic_income", 2025, "east_of_england_mixed"): ( + 6402.9033203125, + 15593.0, + 47190.09765625, + ), + ("abolish_uprating_basic_income", 2025, "london_high_value_owner"): ( + 72988.1015625, + 2080.0, + 109091.8984375, + ), + ("abolish_uprating_basic_income", 2026, "single_renter_london"): ( + 3660.403076171875, + 2184.0, + 23523.59765625, + ), + ("abolish_uprating_basic_income", 2026, "couple_two_children_uc"): ( + 1448.403076171875, + 28977.75, + 45529.34765625, + ), + ("abolish_uprating_basic_income", 2026, "pensioner_couple"): ( + 180.0, + 23872.0, + 23692.0, + ), + ("abolish_uprating_basic_income", 2026, "scottish_lone_parent"): ( + 180.0, + 19434.771484375, + 28254.771484375, + ), + ("abolish_uprating_basic_income", 2026, "welsh_working_family"): ( + 10780.806640625, + 14986.50390625, + 67205.6953125, + ), + ("abolish_uprating_basic_income", 2026, "northern_ireland_owner"): ( + 7472.203125, + 4160.0, + 49687.796875, + ), + ("abolish_uprating_basic_income", 2026, "high_earner_south_east"): ( + 65930.6015625, + 8774.8955078125, + 102844.2890625, + ), + ("abolish_uprating_basic_income", 2026, "unemployed_contributory_jsa"): ( + 180.0, + 5304.0, + 5124.0, + ), + ("abolish_uprating_basic_income", 2026, "esa_and_reported_benefits"): ( + 180.0, + 13104.0, + 24924.0, + ), + ("abolish_uprating_basic_income", 2026, "scottish_pensioner"): ( + 0.0, + 15453.953125, + 15453.953125, + ), + ("abolish_uprating_basic_income", 2026, "large_family_yorkshire"): ( + 3128.403076171875, + 43225.26953125, + 64096.86328125, + ), + ("abolish_uprating_basic_income", 2026, "east_of_england_mixed"): ( + 6408.4033203125, + 15697.0, + 47288.59765625, + ), + ("abolish_uprating_basic_income", 2026, "london_high_value_owner"): ( + 72993.6015625, + 2184.0, + 109190.3984375, + ), + ("abolish_uprating_basic_income", 2028, "single_renter_london"): ( + 3660.403076171875, + 2080.0, + 23419.59765625, + ), + ("abolish_uprating_basic_income", 2028, "couple_two_children_uc"): ( + 1448.403076171875, + 30688.21484375, + 47239.8125, + ), + ("abolish_uprating_basic_income", 2028, "pensioner_couple"): ( + 180.0, + 24052.251953125, + 23872.251953125, + ), + ("abolish_uprating_basic_income", 2028, "scottish_lone_parent"): ( + 180.0, + 20584.078125, + 29404.078125, + ), + ("abolish_uprating_basic_income", 2028, "welsh_working_family"): ( + 10780.806640625, + 15572.53515625, + 67791.7265625, + ), + ("abolish_uprating_basic_income", 2028, "northern_ireland_owner"): ( + 7472.203125, + 4160.0, + 49687.796875, + ), + ("abolish_uprating_basic_income", 2028, "high_earner_south_east"): ( + 66094.6015625, + 8841.5205078125, + 102746.9140625, + ), + ("abolish_uprating_basic_income", 2028, "unemployed_contributory_jsa"): ( + 180.0, + 5200.0, + 5020.0, + ), + ("abolish_uprating_basic_income", 2028, "esa_and_reported_benefits"): ( + 180.0, + 13000.0, + 24820.0, + ), + ("abolish_uprating_basic_income", 2028, "scottish_pensioner"): ( + 0.0, + 15922.6865234375, + 15922.6865234375, + ), + ("abolish_uprating_basic_income", 2028, "large_family_yorkshire"): ( + 3128.403076171875, + 45559.4296875, + 66431.0234375, + ), + ("abolish_uprating_basic_income", 2028, "east_of_england_mixed"): ( + 6408.4033203125, + 15593.0, + 47184.59765625, + ), + ("abolish_uprating_basic_income", 2028, "london_high_value_owner"): ( + 72993.6015625, + 2080.0, + 109086.3984375, + ), +} + + +@pytest.mark.parametrize("reform_name", list(REFORMS)) +@pytest.mark.parametrize("year", YEARS) +def test_budget_totals_match_pinned_values(reform_name, year): + sim = Simulation(situation=situation(year), reform=REFORMS[reform_name] or None) + actual = { + output: [float(value) for value in sim.calculate(output, year)] + for output in OUTPUTS + } + mismatches = [] + for i, household in enumerate(HOUSEHOLDS): + expected = EXPECTED[(reform_name, year, household)] + got = tuple(actual[output][i] for output in OUTPUTS) + if got != expected: + mismatches.append(f"{household}: expected {expected}, got {got}") + assert not mismatches, "\n".join(mismatches) + + +def test_pinned_grid_is_complete(): + assert len(HOUSEHOLDS) >= 13 + assert set(EXPECTED) == { + (reform, year, household) + for reform in REFORMS + for year in YEARS + for household in HOUSEHOLDS + } diff --git a/policyengine_uk/variables/gov/hmrc/household_tax.py b/policyengine_uk/variables/gov/hmrc/household_tax.py index 6a625e5d09..da74a3cae5 100644 --- a/policyengine_uk/variables/gov/hmrc/household_tax.py +++ b/policyengine_uk/variables/gov/hmrc/household_tax.py @@ -1,58 +1,22 @@ from policyengine_uk.model_api import * +from policyengine_uk.programs import budget_variables, budget_variables_in_effect - -HOUSEHOLD_TAX_VARIABLES = [ - "isa_cash_interest_flat_rate_charge", - "expected_sdlt", - "expected_ltt", - "expected_lbtt", - "corporate_sdlt", - "business_rates", - "council_tax", - "high_value_council_tax_surcharge", - "domestic_rates", - "fuel_duty", - "alcohol_duty", - "tobacco_duty", - "gambling_duty", - "vehicle_excise_duty", - "tv_licence", - "wealth_tax", - "non_primary_residence_wealth_tax", - "income_tax", - "national_insurance", - "LVT", - "carbon_tax", - "vat_change", - "capital_gains_tax", - "private_school_vat_change", - "corporate_incident_tax_revenue_change", - "consumer_incident_tax_revenue_change", - "employer_ni_response_capital_incidence", - "employer_ni_response_consumer_incidence", - "student_loan_repayments", -] +# The terms flagged `budget: tax` in programs.yaml, in its budget_order. +HOUSEHOLD_TAX_VARIABLES = budget_variables("tax") class household_tax(Variable): value_type = float entity = Household label = "household taxes" - documentation = "Total taxes owed by the household" + documentation = ( + "Total taxes the household owes: each tax the model counts against " + "the household, less any a reform removes, such as council tax when " + "a reform abolishes it." + ) definition_period = YEAR unit = GBP def formula(household, period, parameters): - abolish_council_tax = parameters.gov.contrib.abolish_council_tax(period) - if abolish_council_tax: - return add( - household, - period, - [ - tax - for tax in HOUSEHOLD_TAX_VARIABLES - if tax not in ["council_tax", "high_value_council_tax_surcharge"] - ], - ) - else: - return add(household, period, HOUSEHOLD_TAX_VARIABLES) + taxes = budget_variables_in_effect("tax", parameters(period)) + return add(household, period, taxes) diff --git a/policyengine_uk/variables/household/income/household_benefits.py b/policyengine_uk/variables/household/income/household_benefits.py index 191050ba30..5a6d2bef89 100644 --- a/policyengine_uk/variables/household/income/household_benefits.py +++ b/policyengine_uk/variables/household/income/household_benefits.py @@ -1,96 +1,36 @@ from policyengine_uk.model_api import * +from policyengine_uk.programs import ( + budget_variables, + budget_variables_in_effect, + uprated_benefit_variables, +) - -HOUSEHOLD_BENEFIT_VARIABLES = [ - "child_benefit", - "council_tax_benefit", - "esa_income", - "esa_contrib", - "housing_benefit", - "income_support", - "jsa_income", - "jsa_contrib", - "pension_credit", - "universal_credit", - "working_tax_credit", - "child_tax_credit", - "attendance_allowance", - "afcs", - "bsp", - "war_widows_pension", - "widowed_parents_allowance", - "carers_allowance", - "dla", - "iidb", - "incapacity_benefit", - "pip", - "sda", - "state_pension", - "maternity_allowance", - "statutory_sick_pay", - "statutory_maternity_pay", - "statutory_paternity_pay", - "ssmg", - "basic_income", - "epg_subsidy", - "cost_of_living_support_payment", - "energy_bills_rebate", - "household_energy_payment", - "winter_fuel_allowance", - "pawhp", - "tax_free_childcare", - "extended_childcare_entitlement", - "universal_childcare_entitlement", - "targeted_childcare_entitlement", - "wales_childcare_offer", - "care_to_learn", - "childcare_grant", - "parents_learning_allowance", - "adult_dependants_grant", - "travel_grant", - "disabled_students_allowance", - "bursary_fund_16_to_19", - "nhs_spending", - "dfe_education_spending", - "dft_subsidy_spending", - "healthy_start_vouchers", - "two_child_limit_payment", - "scottish_child_payment", - "carer_support_payment", - "scottish_carer_supplement", - "disability_basic_income", -] +# The terms flagged `budget: benefit` in programs.yaml, in its budget_order. +HOUSEHOLD_BENEFIT_VARIABLES = budget_variables("benefit") class household_benefits(Variable): value_type = float entity = Household label = "household benefits" - documentation = "Total value of benefits received by household" + documentation = ( + "Total value of the benefits the household receives, less any a " + "reform removes, such as Council Tax Reduction when a reform " + "abolishes council tax, plus each benefit uprating rate a reform sets " + "times the benefits that rate covers." + ) definition_period = YEAR unit = GBP def formula(household, period, parameters): - contrib = parameters(period).gov.contrib - uprating = contrib.benefit_uprating - benefits = HOUSEHOLD_BENEFIT_VARIABLES - if contrib.abolish_council_tax: - benefits = [ - benefit for benefit in benefits if benefit != "council_tax_benefit" - ] + p = parameters(period) + uprating = p.gov.contrib.benefit_uprating + benefits = budget_variables_in_effect("benefit", p) general_benefits = add( - household, - period, - [benefit for benefit in benefits if benefit not in ["basic_income"]], + household, period, uprated_benefit_variables(benefits, "all") ) non_sp_benefits = add( - household, - period, - [ - benefit - for benefit in benefits - if benefit not in ["state_pension", "basic_income"] - ], + household, period, uprated_benefit_variables(benefits, "non_sp") ) return ( add(household, period, benefits)