From d5f9cb66118c0136a3ecef5e8549b6bd1b38c2e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 19:03:29 -0400 Subject: [PATCH 01/19] Keep stored household ISA totals after the projection horizon A dataset that stores cash_isa, stocks_and_shares_isa or household_lifetime_isa_balance lost the total from 2033 (#2740). Past PROJECTION_END_YEAR, policyengine-core runs a variable's adds instead of reading a stored input, and the person balances it sums are absent from such data. Core does not let a variable declare both adds and uprating. In a simulation built from data, Simulation._run_formula now grows the latest earlier stored total by the index its parts declare, the index Core grows the parts by, and runs the sum only where no total is stored. A situation's household total still applies to its own year only. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2740.fixed.md | 1 + policyengine_uk/data/economic_assumptions.py | 30 ++ policyengine_uk/simulation.py | 82 +++-- .../tests/test_stored_household_totals.py | 343 ++++++++++++++++++ .../variables/household/wealth/cash_isa.py | 9 +- .../wealth/household_lifetime_isa_balance.py | 7 +- .../household/wealth/stocks_and_shares_isa.py | 7 +- 7 files changed, 445 insertions(+), 34 deletions(-) create mode 100644 changelog.d/2740.fixed.md create mode 100644 policyengine_uk/tests/test_stored_household_totals.py diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md new file mode 100644 index 0000000000..ef6c3fe805 --- /dev/null +++ b/changelog.d/2740.fixed.md @@ -0,0 +1 @@ +A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after 2032, grown by the index of the person balances it stands for, so means-tested capital no longer drops by the ISA holdings from 2033. diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 815a8d31d1..48e8547154 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -165,10 +165,22 @@ def growth_rate( # which the Enhanced FRS does not store, so it falls to -1); one whose # variable has an `uprating` attribute is scaled by that index; any other, # rent and council tax among them, is held at its last projected value. +# Simulation grows the household totals in STORED_HOUSEHOLD_TOTALS itself. # A finite CGT scenario can shorten the whole dataset horizon; it never # extends beyond the evidence declared by that scenario. PROJECTION_END_YEAR = 2032 +# Household totals that sum person inputs (`adds`) but that a dataset can +# store itself (#2740). For a year with no stored total, Core runs the sum, +# which falls to zero in data that store only the total, and Core does not +# let a variable declare both `adds` and `uprating`. So in a simulation built +# from data, Simulation grows the latest earlier stored total by the index +# its parts declare, as Core grows the parts, and sums the parts only where +# no total is stored. +STORED_HOUSEHOLD_TOTALS = frozenset( + {"cash_isa", "stocks_and_shares_isa", "household_lifetime_isa_balance"} +) + # Before this horizon was extended, Core uprated unmapped inputs from 2030. # Preserve that behavior in the newly materialized years without changing # projections for earlier years. Correcting historical mapping omissions is @@ -194,6 +206,24 @@ def growth_rate( YEAR_SPECIFIC_INPUTS = frozenset({"unused_pension_annual_allowance"}) +def stored_total_uprating(tax_benefit_system, name: str) -> str | None: + """The index that grows a stored household total: its parts' ``uprating``. + + Growing the total like its parts keeps a stored total equal to the sum it + replaces. ``None`` means the parts are held flat. + """ + indices = { + tax_benefit_system.variables[part].uprating + for part in tax_benefit_system.variables[name].adds + } + if len(indices) != 1: + raise ValueError( + f"The parts of {name} declare different uprating indices: " + f"{sorted(map(str, indices))}." + ) + return indices.pop() + + def _unmapped_uprating_indices(uprating): from policyengine_uk.system import system diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 388c26ebc6..bef111f0d5 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -38,8 +38,10 @@ from policyengine_uk.utils.scenario import ConstructionReformState, Scenario from policyengine_uk.data.economic_assumptions import ( BEREAVEMENT_SPLIT_INPUTS, + STORED_HOUSEHOLD_TOTALS, extend_single_year_dataset, reset_growthfactor_uprating, + stored_total_uprating, uprating_input_fingerprint, ) from policyengine_uk.data.dataset_sources import materialize_gcs_dataset_url @@ -698,7 +700,60 @@ def _isa_stored_exemption(self, period): # anchor when translating that stored amount in either direction. return self.calculate(TAX_FREE_INTEREST, period_(period).start.period("year")) + def _record_derived(self, variable_name): + # The value depends on which other periods hold inputs, as an uprated + # one does in Core, so an input set for any period can change it. + history = getattr(self, "_get_store_history", None) + if callable(history): + # Store-history tracking is available in newer Core releases; + # the declared minimum already caches formula results without it. + from policyengine_core.data_storage.store_history import ( + next_sequence_number, + ) + + history().record_derived(variable_name, next_sequence_number()) + + def _index_growth(self, index_name, source, period): + # Core's uprating factor: the ratio of the index at the two starts, + # or 1 where the index has no value or is zero at the source. + if index_name is None: + return 1 + index = self.tax_benefit_system.parameters.get_child(index_name) + source_index = _uprating_index_value(index, source.start) + target_index = _uprating_index_value(index, period.start) + return ( + target_index / source_index + if source_index not in (None, 0) and target_index is not None + else 1 + ) + + def _latest_source(self, name, period): + source = self._isa_latest_input(name, period, earlier=True) + if source is None: + # Core first uprates strictly earlier annual inputs, then carries + # an input starting on the query date if there was none earlier. + source = self._isa_latest_input(name, period) + return source + + def _stored_household_total(self, variable, period): + # A dataset that stores a household total keeps it, projected like + # its parts, in every year (#2740). Core serves the stored total for + # each year the data hold. For a later year it runs the variable's + # sum of person balances, which such data need not hold, so grow the + # latest stored total instead, as Core grows an input with `uprating`. + self._record_derived(variable.name) + source = self._latest_source(variable.name, period) + if source is None: + return None + stored = self.get_holder(variable.name).get_array(source, self.branch_name) + index_name = stored_total_uprating(self.tax_benefit_system, variable.name) + return stored * self._index_growth(index_name, source, period) + def _run_formula(self, variable, population, period): + if variable.name in STORED_HOUSEHOLD_TOTALS and built_from_data(self): + stored = self._stored_household_total(variable, period) + if stored is not None: + return stored result = super()._run_formula(variable, population, period) if ( result is not None @@ -712,20 +767,8 @@ def _run_formula(self, variable, population, period): # This is fallback, after Core's explicit input/abolition checks and # policy formulas. Preserve the old gross history even past the last # materialized dataset year; new additive records use Core normally. - history = getattr(self, "_get_store_history", None) - if callable(history): - # Store-history tracking is available in newer Core releases; - # the declared minimum already caches formula results without it. - from policyengine_core.data_storage.store_history import ( - next_sequence_number, - ) - - history().record_derived(variable.name, next_sequence_number()) - source = self._isa_latest_input(variable.name, period, earlier=True) - if source is None: - # Core first uprates strictly earlier annual inputs, then carries - # an input starting on the query date if there was none earlier. - source = self._isa_latest_input(variable.name, period) + self._record_derived(variable.name) + source = self._latest_source(variable.name, period) holder = self.get_holder(variable.name) if source is None: gross = holder.default_array() @@ -736,16 +779,7 @@ def _run_formula(self, variable, population, period): == "additive" ): gross = gross + self._isa_stored_exemption(source) - if variable.uprating is not None: - index = self.tax_benefit_system.parameters.get_child(variable.uprating) - source_index = _uprating_index_value(index, source.start) - target_index = _uprating_index_value(index, period.start) - factor = ( - target_index / source_index - if source_index not in (None, 0) and target_index is not None - else 1 - ) - gross = gross * factor + gross = gross * self._index_growth(variable.uprating, source, period) return gross - self.calculate(TAX_FREE_INTEREST, period) @staticmethod diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py new file mode 100644 index 0000000000..4d0d63f9f2 --- /dev/null +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -0,0 +1,343 @@ +"""A dataset that stores a household ISA total keeps it in every year (#2740). + +cash_isa, stocks_and_shares_isa and household_lifetime_isa_balance sum person +balances, but a dataset can store the household total instead. Up to +PROJECTION_END_YEAR the dataset projection grows a stored total; after it, +Simulation grows the latest stored total by the index of its parts, the index +each variable's documentation projects it with. A dataset that stores only the +person balances still gets their sum in every year, and so does a situation +for a year it gives no total for. + +Invariants, for any non-negative amounts: + +- a stored total stays positive in every year from 2025 to 2040 when it starts + positive, and from PROJECTION_END_YEAR grows by its declared index; +- with person balances stored instead, the total equals their sum every year; +- the two forms give the same totals and means-test capital when they hold + the same amounts. +""" + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from policyengine_core.periods import period +from policyengine_core.simulations.simulation import _uprating_index_value + +from policyengine_uk import Simulation +from policyengine_uk.data.dataset_schema import UKSingleYearDataset +from policyengine_uk.data.economic_assumptions import ( + PROJECTION_END_YEAR, + STORED_HOUSEHOLD_TOTALS, + _uprating_index_map, + stored_total_uprating, +) + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +DATA_YEAR = 2025 +YEARS = range(DATA_YEAR, 2041) +CAPITAL_YEARS = (DATA_YEAR, PROJECTION_END_YEAR, PROJECTION_END_YEAR + 1, YEARS[-1]) +# Each total, the person balance it sums, and the variable its documentation +# says it is projected with. +TOTALS = { + "cash_isa": ("cash_isa_balance", "savings"), + "stocks_and_shares_isa": ("stocks_and_shares_isa_balance", "corporate_wealth"), + "household_lifetime_isa_balance": ("lifetime_isa_balance", "lifetime_isa_balance"), +} +MEANS_TEST_CAPITAL = ( + "uc_assessable_capital", + "housing_benefit_assessable_capital", + "pension_credit_assessable_capital", + "income_support_assessable_capital", + "jsa_income_assessable_capital", + "esa_income_assessable_capital", + "council_tax_reduction_assessable_capital", +) +PROPERTY_SETTINGS = settings( + max_examples=8, + deadline=None, + derandomize=True, + database=None, +) + + +def _dataset(households, person_columns=(), household_columns=()): + """Two adults per household, with the named columns from ``households``. + + Each household is a dict of amounts. A person column ``name`` takes + ``name`` split between the two adults by the household's ``share``. + """ + people = [] + for index, household in enumerate(households): + share = household.get("share", 1.0) + for adult, age in ((0, 30), (1, 28)): + person = { + "person_id": 2 * index + adult + 1, + "person_benunit_id": index + 1, + "person_household_id": index + 1, + "age": age, + } + for name in person_columns: + first = round(household[name] * share, 2) + person[name] = first if adult == 0 else household[name] - first + people.append(person) + count = len(households) + return UKSingleYearDataset( + person=pd.DataFrame(people), + benunit=pd.DataFrame({"benunit_id": range(1, count + 1)}), + household=pd.DataFrame( + { + "household_id": range(1, count + 1), + "household_weight": [1.0] * count, + "region": ["LONDON"] * count, + "tenure_type": ["RENT_PRIVATELY"] * count, + "council_tax": [1_500.0] * count, + "rent": [12_000.0] * count, + "savings": [1_000.0] * count, + **{ + name: [float(household[name]) for household in households] + for name in household_columns + }, + } + ), + fiscal_year=DATA_YEAR, + ) + + +def _values(simulation, name, years=YEARS): + return {year: simulation.calculate(name, year).astype(float) for year in years} + + +def _growth(simulation, name, year): + """Growth from PROJECTION_END_YEAR to ``year`` by the index ``name`` declares.""" + index = simulation.tax_benefit_system.parameters.get_child( + simulation.tax_benefit_system.variables[name].uprating + ) + return _uprating_index_value(index, period(year).start) / _uprating_index_value( + index, period(PROJECTION_END_YEAR).start + ) + + +def test_stored_household_totals_grow_past_the_projection_horizon(): + simulation = Simulation( + dataset=_dataset( + [ + { + "cash_isa": 3_000.0, + "stocks_and_shares_isa": 2_000.0, + "household_lifetime_isa_balance": 4_000.0, + } + ], + household_columns=TOTALS, + ) + ) + capital = _values(simulation, "uc_assessable_capital") + savings = _values(simulation, "savings") + for name, (_, projected_with) in TOTALS.items(): + totals = _values(simulation, name) + for year in YEARS: + if year <= PROJECTION_END_YEAR: + continue + growth = _growth(simulation, projected_with, year) + assert growth > 1 + assert totals[year][0] == pytest.approx( + totals[PROJECTION_END_YEAR][0] * growth, rel=1e-6 + ) + # Savings, the cash ISA and the stocks-and-shares ISA (through + # share_like_capital) make up the capital, each grown alike. + for year in YEARS: + assert capital[year][0] == pytest.approx(6 * savings[year][0], rel=1e-5) + + +def test_stored_totals_project_exactly_like_what_they_are_projected_with(): + # cash_isa is stored equal to savings, stocks_and_shares_isa to + # corporate_wealth, and household_lifetime_isa_balance to the one + # lifetime_isa_balance in the household: Core projects the second of each + # pair, Simulation the first. + household = { + "cash_isa": 1_000.0, + "stocks_and_shares_isa": 2_345.67, + "corporate_wealth": 2_345.67, + "household_lifetime_isa_balance": 4_321.09, + "lifetime_isa_balance": 4_321.09, + } + simulation = Simulation( + dataset=_dataset( + [household], + person_columns=["lifetime_isa_balance"], + household_columns=[ + "cash_isa", + "stocks_and_shares_isa", + "corporate_wealth", + "household_lifetime_isa_balance", + ], + ) + ) + for name, other in ( + ("cash_isa", "savings"), + ("stocks_and_shares_isa", "corporate_wealth"), + ("household_lifetime_isa_balance", "lifetime_isa_balance"), + ): + for year in YEARS: + total = simulation.calculate(name, year) + assert total.sum() == simulation.calculate(other, year).sum(), (name, year) + # So the stocks-and-shares ISA stays within corporate_wealth. + for year in YEARS: + share_like = simulation.calculate("share_like_capital", year) + assert share_like[0] == simulation.calculate("corporate_wealth", year)[0] + + +def test_person_balances_still_sum_without_a_stored_total(): + simulation = Simulation( + dataset=_dataset( + [ + { + "cash_isa_balance": 3_000.0, + "stocks_and_shares_isa_balance": 2_000.0, + "lifetime_isa_balance": 4_000.0, + "share": 0.25, + } + ], + person_columns=[part for part, _ in TOTALS.values()], + ) + ) + for name, (part, _) in TOTALS.items(): + for year in YEARS: + balances = simulation.calculate(part, year) + assert balances[0] > 0 and balances[1] > 0 + assert simulation.calculate(name, year)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ) + + +def test_a_situation_total_is_used_only_for_the_year_it_is_given_for(): + simulation = Simulation( + situation={ + "people": { + "adult": { + "age": {DATA_YEAR: 40}, + "cash_isa_balance": {DATA_YEAR: 3_000}, + } + }, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "cash_isa": {DATA_YEAR: 7_000}, + } + }, + } + ) + assert simulation.calculate("cash_isa", DATA_YEAR)[0] == 7_000 + for year in (DATA_YEAR + 1, PROJECTION_END_YEAR + 1): + balance = simulation.calculate("cash_isa_balance", year)[0] + assert balance > 3_000 + assert simulation.calculate("cash_isa", year)[0] == balance + + +def test_a_grown_total_depends_on_every_year_that_holds_a_total(): + # Like an input Core uprates, a grown total changes when an input is set + # for any year, so Core's branch invalidation must see it as derived. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + later = PROJECTION_END_YEAR + 3 + simulation.calculate("cash_isa", later) + simulation.calculate("savings", later) + history = simulation._get_store_history() + for name in ("cash_isa", "savings"): + assert history.earliest_dependency(name, period(later - 1)) is not None + + +def test_stored_totals_grow_by_the_index_they_are_projected_with(): + from policyengine_uk.system import system + + for name, (part, projected_with) in TOTALS.items(): + assert system.variables[name].adds == [part] + index = system.variables[projected_with].uprating + assert index is not None + assert stored_total_uprating(system, name) == index + + +def test_every_projected_household_sum_of_uprated_inputs_is_a_stored_total(): + # A total listed for dataset projection whose parts Core uprates must keep + # a stored value past PROJECTION_END_YEAR too. + from policyengine_uk.system import system + + listed = {name for names in _uprating_index_map().values() for name in names} + sums_of_uprated_inputs = { + name + for name in listed + if isinstance(system.variables[name].adds, list) + and not system.variables[name].formulas + and all( + system.variables[part].is_input_variable() + and system.variables[part].uprating is not None + for part in system.variables[name].adds + ) + } + assert sums_of_uprated_inputs == STORED_HOUSEHOLD_TOTALS + + +AMOUNTS = st.integers(min_value=0, max_value=200_000_000).map(lambda p: p / 100) +HOUSEHOLDS = st.lists( + st.fixed_dictionaries( + { + "cash_isa": AMOUNTS, + "stocks_and_shares_isa": AMOUNTS, + "household_lifetime_isa_balance": AMOUNTS, + "share": st.sampled_from([0.0, 0.3, 0.5, 1.0]), + } + ), + min_size=1, + max_size=4, +) + + +@PROPERTY_SETTINGS +@given(households=HOUSEHOLDS) +def test_stored_totals_and_person_balances_agree_in_every_year(households): + # The totals-only dataset stores the three household totals; it also + # stores the Lifetime ISA balances, which the means tests read from the + # person. The balances-only dataset stores the same amounts per person. + households = [ + {**household, **{part: household[name] for name, (part, _) in TOTALS.items()}} + for household in households + ] + totals_only = Simulation( + dataset=_dataset( + households, + person_columns=["lifetime_isa_balance"], + household_columns=TOTALS, + ) + ) + balances_only = Simulation( + dataset=_dataset( + households, person_columns=[part for part, _ in TOTALS.values()] + ) + ) + members = balances_only.calculate("person_household_id", DATA_YEAR) + for name, (part, projected_with) in TOTALS.items(): + stored = np.array([household[name] for household in households]) + totals = _values(totals_only, name) + summed = _values(balances_only, name) + for year in YEARS: + assert ((totals[year] > 0) == (stored > 0)).all(), (name, year) + balances = balances_only.calculate(part, year) + sums = np.bincount(members - 1, weights=balances, minlength=len(stored)) + assert summed[year] == pytest.approx(sums, rel=1e-6, abs=1e-6) + assert totals[year] == pytest.approx(summed[year], rel=1e-6, abs=1e-6) + if year > PROJECTION_END_YEAR: + growth = _growth(totals_only, projected_with, year) + assert totals[year] == pytest.approx( + totals[PROJECTION_END_YEAR] * growth, rel=1e-6 + ) + # Capital reads the totals compared above in every year; these years + # cover the data year, both sides of the horizon and the last year. + for name in MEANS_TEST_CAPITAL: + for year in CAPITAL_YEARS: + assert totals_only.calculate(name, year) == pytest.approx( + balances_only.calculate(name, year), rel=1e-5, abs=1e-3 + ), (name, year) diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index 968a74544e..a1bd926ad4 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -6,10 +6,11 @@ class cash_isa(Variable): documentation = ( "Amount held in cash Individual Savings Accounts: cash_isa_balance " "summed over the household's members. A dataset that stores this " - "column keeps the stored value, and projects it with savings: it is " - "recomputed only when not stored, so a reform or situation that edits " - "cash_isa_balance leaves a stored total unchanged. A household input " - "is used for the year it is given for only." + "column keeps the stored value, and projects it with savings in every " + "year: it is recomputed only when not stored, so a reform or situation " + "that edits cash_isa_balance leaves a stored total unchanged. A " + "household input in a situation is used for the year it is given for " + "only." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py index 2ec53080cb..68d07a4234 100644 --- a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py +++ b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py @@ -7,9 +7,10 @@ class household_lifetime_isa_balance(Variable): "Lifetime ISA balances summed over the household's members. Like " "lifetime_isa_balance, it is a component of gross_financial_wealth and " "must never be added to it. A dataset that stores this column keeps " - "the stored value: it is recomputed only when not stored, so a reform " - "or situation that edits lifetime_isa_balance leaves it unchanged. No " - "means test reads it." + "the stored value, and projects it with lifetime_isa_balance in every " + "year: it is recomputed only when not stored, so a reform or situation " + "that edits lifetime_isa_balance leaves it unchanged. No means test " + "reads it." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index d91949f784..f26d60f224 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -8,10 +8,11 @@ class stocks_and_shares_isa(Variable): "Accounts: stocks_and_shares_isa_balance summed over the household's " "members. Part of corporate_wealth, so the two must never be summed. " "A dataset that stores this column keeps the stored value, and " - "projects it with corporate_wealth: it is recomputed only when not " - "stored, so a reform or situation that edits " + "projects it with corporate_wealth in every year: it is recomputed " + "only when not stored, so a reform or situation that edits " "stocks_and_shares_isa_balance leaves a stored total unchanged. A " - "household input is used for the year it is given for only." + "household input in a situation is used for the year it is given for " + "only." ) entity = Household definition_period = YEAR From 1f16ecaf5b7c175494437e83f5e65171393a4e00 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 19:03:38 -0400 Subject: [PATCH 02/19] Document how stored ISA totals project past 2032 Co-Authored-By: Claude Opus 5.5 --- docs/book/assumptions/isa-inputs-and-capital-flows.md | 10 ++++++---- docs/engineering/skills/dataset-sources.md | 11 +++++++---- policyengine_uk/data/uprating_indices.yaml | 3 ++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index b64209bcd7..daab0c8f92 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -100,10 +100,12 @@ established by the household tests. `cash_isa_balance`, and `stocks_and_shares_isa` sums their `stocks_and_shares_isa_balance`, as `household_lifetime_isa_balance` sums `lifetime_isa_balance`. A dataset or situation that stores a household total -keeps it: the total is recomputed only when it is not stored, and a stored -total is projected with `savings` or `corporate_wealth`. A household input is -used only for the year it is given for; the person balances carry the -variable-level uprating to later years. The means tests read the household +keeps it: the total is recomputed only when it is not stored. The model +projects a total that a dataset stores with `savings` or `corporate_wealth` in +every year, including years after the last one the dataset projection +reaches. A household input in a situation is used only for the year it is +given for; the person balances carry the variable-level uprating to later +years. The means tests read the household totals, so a person's balance is pooled with the household's other financial capital rather than attributed to their own benefit unit. diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 4a4c199baa..679eadf2e3 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -156,10 +156,13 @@ Producers can export per-person `cash_isa_balance` and `stocks_and_shares_isa_balance`. The household `cash_isa` and `stocks_and_shares_isa` sum them, but a stored household column takes precedence, so existing datasets that store only the household totals keep -them, projected with `savings` and `corporate_wealth` through -`uprating_indices.yaml`. A producer that exports the person balances should -either stop exporting the household totals or keep them equal to the sums; -the model does not reconcile the two. +them in every year. `uprating_indices.yaml` projects them with `savings` and +`corporate_wealth` up to `PROJECTION_END_YEAR`; after it, the simulation grows +the last stored total by the index its person balances declare +(`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also +covers `household_lifetime_isa_balance`). A producer that exports the person +balances should either stop exporting the household totals or keep them +equal to the sums; the model does not reconcile the two. `cash_isa_subscriptions` and `stocks_and_shares_isa_subscriptions` are annual person flows, summed over all of a person's accounts of each type, that diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index 2f92d6e6fc..8cff45ecb7 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -112,7 +112,8 @@ gov.economic_assumptions.yoy_growth.obr.per_capita.gdp: - benunit_reported_capital # cash_isa and stocks_and_shares_isa sum the person balances listed with # them; they are listed so a dataset that stores the household totals -# grows them like their parts. +# grows them like their parts. Past PROJECTION_END_YEAR, Simulation grows a +# stored total the same way (STORED_HOUSEHOLD_TOTALS). - cash_isa - cash_isa_balance - corporate_wealth From 8a2ab9d95551321fffa7f41df17d9b9ed0a204f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 19:46:02 -0400 Subject: [PATCH 03/19] Check stored totals stay flat with savings without economic growth Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_stored_household_totals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index 4d0d63f9f2..b0cc1487dc 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -33,6 +33,7 @@ _uprating_index_map, stored_total_uprating, ) +from policyengine_uk.scenarios import no_economic_assumptions pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -151,11 +152,14 @@ def test_stored_household_totals_grow_past_the_projection_horizon(): assert capital[year][0] == pytest.approx(6 * savings[year][0], rel=1e-5) -def test_stored_totals_project_exactly_like_what_they_are_projected_with(): +@pytest.mark.parametrize( + "scenario", [None, no_economic_assumptions], ids=["current_law", "no_growth"] +) +def test_stored_totals_project_exactly_like_what_they_are_projected_with(scenario): # cash_isa is stored equal to savings, stocks_and_shares_isa to # corporate_wealth, and household_lifetime_isa_balance to the one # lifetime_isa_balance in the household: Core projects the second of each - # pair, Simulation the first. + # pair, Simulation the first. Without economic growth both stay flat. household = { "cash_isa": 1_000.0, "stocks_and_shares_isa": 2_345.67, @@ -164,6 +168,7 @@ def test_stored_totals_project_exactly_like_what_they_are_projected_with(): "lifetime_isa_balance": 4_321.09, } simulation = Simulation( + scenario=scenario, dataset=_dataset( [household], person_columns=["lifetime_isa_balance"], @@ -173,7 +178,7 @@ def test_stored_totals_project_exactly_like_what_they_are_projected_with(): "corporate_wealth", "household_lifetime_isa_balance", ], - ) + ), ) for name, other in ( ("cash_isa", "savings"), From ac21e3c1abe73c0476dcad294e3cc230f4cf3189 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 20:23:30 -0400 Subject: [PATCH 04/19] Let newer person balances replace an older stored ISA total A stored household total now grows only while no person balances are stored for a later year; balances stored after the total are the newer observation, so their sum stands, as on main. The override also leaves a variable to Core when a reform gives it a formula or drops its sum, and records the parts as well as the total as derived. Tests check this through a branch edit instead of Core's store history, pin that an input frame counts as data, and build fewer, larger property examples. The docs name the Lifetime ISA total and the projection's capital gains horizon, and use active voice. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2740.fixed.md | 2 +- .../isa-inputs-and-capital-flows.md | 10 +- docs/engineering/skills/dataset-sources.md | 16 ++- policyengine_uk/data/economic_assumptions.py | 5 +- policyengine_uk/data/uprating_indices.yaml | 5 +- policyengine_uk/simulation.py | 19 ++- .../tests/test_stored_household_totals.py | 119 +++++++++++++++--- .../variables/household/wealth/cash_isa.py | 3 +- .../household/wealth/stocks_and_shares_isa.py | 3 +- 9 files changed, 142 insertions(+), 40 deletions(-) diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md index ef6c3fe805..4eb6e9fcae 100644 --- a/changelog.d/2740.fixed.md +++ b/changelog.d/2740.fixed.md @@ -1 +1 @@ -A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after 2032, grown by the index of the person balances it stands for, so means-tested capital no longer drops by the ISA holdings from 2033. +A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it in later years, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset stores those balances for a later year. Means-tested capital no longer drops by the ISA holdings from 2033. diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index daab0c8f92..57b39f16ca 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -101,11 +101,11 @@ established by the household tests. `stocks_and_shares_isa_balance`, as `household_lifetime_isa_balance` sums `lifetime_isa_balance`. A dataset or situation that stores a household total keeps it: the total is recomputed only when it is not stored. The model -projects a total that a dataset stores with `savings` or `corporate_wealth` in -every year, including years after the last one the dataset projection -reaches. A household input in a situation is used only for the year it is -given for; the person balances carry the variable-level uprating to later -years. The means tests read the household +projects a total that a dataset stores with `savings`, `corporate_wealth` or +`lifetime_isa_balance` in every year, including years after the last one the +dataset projection reaches, unless the dataset stores the person balances for +a later year. A household input in a situation applies only to its own year; +the person balances carry the variable-level uprating to later years. The means tests read the household totals, so a person's balance is pooled with the household's other financial capital rather than attributed to their own benefit unit. diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 679eadf2e3..a60616d42e 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -157,12 +157,16 @@ Producers can export per-person `cash_isa_balance` and `stocks_and_shares_isa` sum them, but a stored household column takes precedence, so existing datasets that store only the household totals keep them in every year. `uprating_indices.yaml` projects them with `savings` and -`corporate_wealth` up to `PROJECTION_END_YEAR`; after it, the simulation grows -the last stored total by the index its person balances declare -(`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also -covers `household_lifetime_isa_balance`). A producer that exports the person -balances should either stop exporting the household totals or keep them -equal to the sums; the model does not reconcile the two. +`corporate_wealth` through the years the dataset projection covers. In later +years, and in any year a data source such as an input frame holds no total +for, the simulation grows the latest stored total by the index its person +balances declare (`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, +which also covers `household_lifetime_isa_balance`). Person balances stored +for a later year than the total replace it with their sum. Situations differ: +a household total given in one applies only to its own year, so a situation +reloaded from its input frame carries such a total forward. A producer that +exports the person balances should either stop exporting the household totals +or keep them equal to the sums; the model does not reconcile the two. `cash_isa_subscriptions` and `stocks_and_shares_isa_subscriptions` are annual person flows, summed over all of a person's accounts of each type, that diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 48e8547154..a5c6e8cc31 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -175,8 +175,9 @@ def growth_rate( # which falls to zero in data that store only the total, and Core does not # let a variable declare both `adds` and `uprating`. So in a simulation built # from data, Simulation grows the latest earlier stored total by the index -# its parts declare, as Core grows the parts, and sums the parts only where -# no total is stored. +# its parts declare, as Core grows the parts. It sums the parts where no +# total is stored for that year or an earlier one, or where the parts are +# stored for a later year than the total. STORED_HOUSEHOLD_TOTALS = frozenset( {"cash_isa", "stocks_and_shares_isa", "household_lifetime_isa_balance"} ) diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index 8cff45ecb7..43e43886c9 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -112,8 +112,9 @@ gov.economic_assumptions.yoy_growth.obr.per_capita.gdp: - benunit_reported_capital # cash_isa and stocks_and_shares_isa sum the person balances listed with # them; they are listed so a dataset that stores the household totals -# grows them like their parts. Past PROJECTION_END_YEAR, Simulation grows a -# stored total the same way (STORED_HOUSEHOLD_TOTALS). +# grows them like their parts. Past the projected years, Simulation grows +# each stored total in STORED_HOUSEHOLD_TOTALS, household_lifetime_isa_balance +# among them, the same way. - cash_isa - cash_isa_balance - corporate_wealth diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index bef111f0d5..dcb01017b5 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -741,16 +741,31 @@ def _stored_household_total(self, variable, period): # each year the data hold. For a later year it runs the variable's # sum of person balances, which such data need not hold, so grow the # latest stored total instead, as Core grows an input with `uprating`. - self._record_derived(variable.name) + # Which years hold the total or its parts decides the result. + for name in (variable.name, *variable.adds): + self._record_derived(name) source = self._latest_source(variable.name, period) if source is None: return None + for part in variable.adds: + part_source = self._isa_latest_input(part, period) + if part_source is not None and part_source.start > source.start: + # Person balances stored for a later year than the total are + # the newer observation, so their sum stands. + return None stored = self.get_holder(variable.name).get_array(source, self.branch_name) index_name = stored_total_uprating(self.tax_benefit_system, variable.name) return stored * self._index_growth(index_name, source, period) def _run_formula(self, variable, population, period): - if variable.name in STORED_HOUSEHOLD_TOTALS and built_from_data(self): + # Only the sum: a reform that gives the variable a formula, or makes + # it a plain input, leaves it to Core. + if ( + variable.name in STORED_HOUSEHOLD_TOTALS + and variable.adds + and variable.get_formula(period) is None + and built_from_data(self) + ): stored = self._stored_household_total(variable, period) if stored is not None: return stored diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index b0cc1487dc..37b67c523e 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -5,8 +5,9 @@ PROJECTION_END_YEAR the dataset projection grows a stored total; after it, Simulation grows the latest stored total by the index of its parts, the index each variable's documentation projects it with. A dataset that stores only the -person balances still gets their sum in every year, and so does a situation -for a year it gives no total for. +person balances, or stores them for a later year than the total, gets their +sum, and so does a situation for a year it gives no total for. An input frame +is data, so a total it stores grows too. Invariants, for any non-negative amounts: @@ -26,7 +27,10 @@ from policyengine_core.simulations.simulation import _uprating_index_value from policyengine_uk import Simulation -from policyengine_uk.data.dataset_schema import UKSingleYearDataset +from policyengine_uk.data.dataset_schema import ( + UKMultiYearDataset, + UKSingleYearDataset, +) from policyengine_uk.data.economic_assumptions import ( PROJECTION_END_YEAR, STORED_HOUSEHOLD_TOTALS, @@ -57,14 +61,14 @@ "council_tax_reduction_assessable_capital", ) PROPERTY_SETTINGS = settings( - max_examples=8, + max_examples=5, deadline=None, derandomize=True, database=None, ) -def _dataset(households, person_columns=(), household_columns=()): +def _dataset(households, person_columns=(), household_columns=(), year=DATA_YEAR): """Two adults per household, with the named columns from ``households``. Each household is a dict of amounts. A person column ``name`` takes @@ -103,7 +107,7 @@ def _dataset(households, person_columns=(), household_columns=()): }, } ), - fiscal_year=DATA_YEAR, + fiscal_year=year, ) @@ -111,13 +115,13 @@ def _values(simulation, name, years=YEARS): return {year: simulation.calculate(name, year).astype(float) for year in years} -def _growth(simulation, name, year): - """Growth from PROJECTION_END_YEAR to ``year`` by the index ``name`` declares.""" +def _growth(simulation, name, year, base=PROJECTION_END_YEAR): + """Growth from ``base`` to ``year`` by the index ``name`` declares.""" index = simulation.tax_benefit_system.parameters.get_child( simulation.tax_benefit_system.variables[name].uprating ) return _uprating_index_value(index, period(year).start) / _uprating_index_value( - index, period(PROJECTION_END_YEAR).start + index, period(base).start ) @@ -242,18 +246,97 @@ def test_a_situation_total_is_used_only_for_the_year_it_is_given_for(): assert simulation.calculate("cash_isa", year)[0] == balance -def test_a_grown_total_depends_on_every_year_that_holds_a_total(): - # Like an input Core uprates, a grown total changes when an input is set - # for any year, so Core's branch invalidation must see it as derived. +def test_a_total_set_on_a_branch_replaces_the_one_a_later_year_grew_from(): + # Like savings, which Core uprates, a grown total depends on which years + # hold a total, so a total set on a branch for a year in between reaches + # a later year the branch has already calculated. simulation = Simulation( dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) ) - later = PROJECTION_END_YEAR + 3 - simulation.calculate("cash_isa", later) - simulation.calculate("savings", later) - history = simulation._get_store_history() + branch = simulation.get_branch("edited") + between, later = PROJECTION_END_YEAR + 1, PROJECTION_END_YEAR + 3 + for name in ("cash_isa", "savings"): + assert branch.calculate(name, later)[0] > 0 + branch.set_input(name, between, np.array([10_000.0])) + growth = _growth(simulation, "savings", later, base=between) for name in ("cash_isa", "savings"): - assert history.earliest_dependency(name, period(later - 1)) is not None + assert branch.calculate(name, later)[0] == pytest.approx( + 10_000 * growth, rel=1e-6 + ) + assert simulation.calculate("cash_isa", later)[0] == pytest.approx( + 3 * simulation.calculate("savings", later)[0], rel=1e-6 + ) + + +def test_person_balances_stored_after_the_total_are_summed_instead(): + # A total and balances that a dataset stores for the same year: the total + # takes precedence, and grows. Balances stored for later years than the + # total are the newer observation, so their sum stands. + def year(columns, person_columns=(), share=0.25, year=DATA_YEAR): + return _dataset( + [{**columns, "share": share}], + person_columns=person_columns, + household_columns=[name for name in columns if name == "cash_isa"], + year=year, + ) + + simulation = Simulation( + dataset=UKMultiYearDataset( + datasets=[ + year( + {"cash_isa": 5_000.0, "cash_isa_balance": 400.0}, + ["cash_isa_balance"], + ), + year({"cash_isa_balance": 600.0}, ["cash_isa_balance"], year=2026), + year({"cash_isa_balance": 800.0}, ["cash_isa_balance"], year=2027), + ] + ) + ) + assert simulation.calculate("cash_isa", DATA_YEAR)[0] == 5_000 + assert simulation.calculate("cash_isa", 2026)[0] == pytest.approx(600) + assert simulation.calculate("cash_isa", 2027)[0] == pytest.approx(800) + balances = simulation.calculate("cash_isa_balance", 2028) + assert balances.sum() > 800 + assert simulation.calculate("cash_isa", 2028)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ) + + same_year = Simulation( + dataset=UKMultiYearDataset( + datasets=[ + year( + {"cash_isa": 5_000.0, "cash_isa_balance": 400.0}, + ["cash_isa_balance"], + ), + ] + ) + ) + growth = _growth(same_year, "savings", 2026, base=DATA_YEAR) + assert same_year.calculate("cash_isa", 2026)[0] == pytest.approx( + 5_000 * growth, rel=1e-6 + ) + + +def test_a_total_stored_in_an_input_frame_grows_like_savings(): + # An input frame is data: a total it stores grows into later years, as a + # total in a situation does not. + frame = pd.DataFrame( + { + "person_id__2025": [1, 2], + "person_benunit_id__2025": [1, 1], + "person_household_id__2025": [1, 1], + "age__2025": [30, 28], + "savings__2025": [1_000.0, 1_000.0], + "cash_isa__2025": [3_000.0, 3_000.0], + } + ) + simulation = Simulation(dataset=frame) + for year in (DATA_YEAR + 1, PROJECTION_END_YEAR + 1): + savings = simulation.calculate("savings", year)[0] + assert savings > 1_000 + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + 3 * savings, rel=1e-6 + ) def test_stored_totals_grow_by_the_index_they_are_projected_with(): @@ -297,7 +380,7 @@ def test_every_projected_household_sum_of_uprated_inputs_is_a_stored_total(): } ), min_size=1, - max_size=4, + max_size=6, ) diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index a1bd926ad4..6c1e5af29a 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -9,8 +9,7 @@ class cash_isa(Variable): "column keeps the stored value, and projects it with savings in every " "year: it is recomputed only when not stored, so a reform or situation " "that edits cash_isa_balance leaves a stored total unchanged. A " - "household input in a situation is used for the year it is given for " - "only." + "household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index f26d60f224..140420edd9 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -11,8 +11,7 @@ class stocks_and_shares_isa(Variable): "projects it with corporate_wealth in every year: it is recomputed " "only when not stored, so a reform or situation that edits " "stocks_and_shares_isa_balance leaves a stored total unchanged. A " - "household input in a situation is used for the year it is given for " - "only." + "household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR From 3b458e9ede05db97f54adafbb0ca33d6fd9210e6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 20:25:32 -0400 Subject: [PATCH 05/19] Check that a reform's formula for an ISA total runs past the stored years Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stored_household_totals.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index 37b67c523e..ad085363ea 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -24,6 +24,7 @@ from hypothesis import given, settings from hypothesis import strategies as st from policyengine_core.periods import period +from policyengine_core.reforms import Reform from policyengine_core.simulations.simulation import _uprating_index_value from policyengine_uk import Simulation @@ -37,6 +38,7 @@ _uprating_index_map, stored_total_uprating, ) +from policyengine_uk.model_api import YEAR, Household, Variable from policyengine_uk.scenarios import no_economic_assumptions pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -339,6 +341,33 @@ def test_a_total_stored_in_an_input_frame_grows_like_savings(): ) +def test_a_reform_formula_for_a_total_runs_in_years_with_no_stored_total(): + # A reform that gives a total a formula replaces its sum, so Core runs the + # formula wherever no total is stored, as for any formula variable. + class cash_isa(Variable): + value_type = float + entity = Household + label = "cash ISA holdings, twice savings" + definition_period = YEAR + + def formula(household, period, parameters): + return 2 * household("savings", period) + + class twice_savings(Reform): + def apply(self): + self.update_variable(cash_isa) + + simulation = Simulation( + reform=twice_savings, + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]), + ) + for year, multiple in ((PROJECTION_END_YEAR, 3), (PROJECTION_END_YEAR + 1, 2)): + savings = simulation.calculate("savings", year)[0] + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + multiple * savings, rel=1e-6 + ) + + def test_stored_totals_grow_by_the_index_they_are_projected_with(): from policyengine_uk.system import system From 8991161a31b4bbf40c97f1f71f48c87bf3e56aff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 20:28:50 -0400 Subject: [PATCH 06/19] Rewrap the ISA capital stocks paragraph Co-Authored-By: Claude Opus 5.5 --- docs/book/assumptions/isa-inputs-and-capital-flows.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index 57b39f16ca..3823214819 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -105,9 +105,10 @@ projects a total that a dataset stores with `savings`, `corporate_wealth` or `lifetime_isa_balance` in every year, including years after the last one the dataset projection reaches, unless the dataset stores the person balances for a later year. A household input in a situation applies only to its own year; -the person balances carry the variable-level uprating to later years. The means tests read the household -totals, so a person's balance is pooled with the household's other financial -capital rather than attributed to their own benefit unit. +the person balances carry the variable-level uprating to later years. The +means tests read the household totals, so a person's balance is pooled with +the household's other financial capital rather than attributed to their own +benefit unit. Ordinary accessible cash ISA balances enter the capital sources for Universal Credit, Housing Benefit, Pension Credit, Income Support, income-based JSA and From 2b7ccc1e76f8fbbc64b6511257d0150ac8adee15 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 23:20:46 -0400 Subject: [PATCH 07/19] Leave a reform's parameter sum for an ISA total to Core Core lets `adds` name a parameter that lists the parts, or list a parameter beside variables. The stored-total path read each part as a variable, so a reform that summed cash_isa that way raised VariableNotFoundError (for 's', the first character of the path) once a dataset stored the total. Simulation now grows a stored total only when `adds` lists variables, and otherwise runs Core's sum, as it does for a reform formula. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/simulation.py | 7 ++- .../tests/test_stored_household_totals.py | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index b1cef39954..c402ec84f1 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -766,11 +766,14 @@ def _stored_household_total(self, variable, period): return stored * self._index_growth(index_name, source, period) def _run_formula(self, variable, population, period): - # Only the sum: a reform that gives the variable a formula, or makes - # it a plain input, leaves it to Core. + # Only a sum of listed variables: a reform that gives the variable a + # formula, sums a parameter or its list, or makes the variable a + # plain input, leaves it to Core. if ( variable.name in STORED_HOUSEHOLD_TOTALS and variable.adds + and not isinstance(variable.adds, str) + and all(part in self.tax_benefit_system.variables for part in variable.adds) and variable.get_formula(period) is None and built_from_data(self) ): diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index ad085363ea..41e3a725c3 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -23,6 +23,7 @@ import pytest from hypothesis import given, settings from hypothesis import strategies as st +from policyengine_core.parameters import ParameterNode from policyengine_core.periods import period from policyengine_core.reforms import Reform from policyengine_core.simulations.simulation import _uprating_index_value @@ -368,6 +369,56 @@ def apply(self): ) +@pytest.mark.parametrize( + "reform_adds", + ["stored_totals_test.parts", ["cash_isa_balance", "stored_totals_test.extra"]], + ids=["parameter", "list_with_a_parameter"], +) +def test_a_reform_sum_through_parameters_runs_in_years_with_no_stored_total( + reform_adds, +): + # Core lets `adds` name a parameter that lists the parts, or list a + # parameter beside variables. A reform that sums a total that way replaces + # its baseline sum, so Core runs it wherever no total is stored. + class cash_isa(Variable): + value_type = float + entity = Household + label = "cash ISA holdings, summed through parameters" + definition_period = YEAR + adds = reform_adds + + class parameter_sum(Reform): + def apply(self): + self.parameters.add_child( + "stored_totals_test", + ParameterNode( + "stored_totals_test", + data={ + "parts": {"values": {"2000-01-01": ["cash_isa_balance"]}}, + "extra": {"values": {"2000-01-01": 100}}, + }, + ), + ) + self.update_variable(cash_isa) + + simulation = Simulation( + reform=parameter_sum, + dataset=_dataset( + [{"cash_isa": 5_000.0, "cash_isa_balance": 400.0, "share": 0.25}], + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ), + ) + extra = 0 if isinstance(reform_adds, str) else 100 + later = PROJECTION_END_YEAR + 1 + assert simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] > 5_000 + balances = simulation.calculate("cash_isa_balance", later) + assert balances.sum() > 400 + assert simulation.calculate("cash_isa", later)[0] == pytest.approx( + balances.sum() + extra, rel=1e-6 + ) + + def test_stored_totals_grow_by_the_index_they_are_projected_with(): from policyengine_uk.system import system From 10b19a53c3345ead4ebb7dbf82910abded30aba6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 23:50:57 -0400 Subject: [PATCH 08/19] Leave a reform sum whose parts share no index to Core A reform that gives cash_isa a part with another uprating index, or none, left stored_total_uprating no single index, and its ValueError stopped the simulation once a dataset stored the total. Simulation now runs Core's sum in that case, as for a reform that sums a parameter. Simulation also records the parts as derived only when it returns a grown total, the one result that reads none of them, and a test checks that a balance set on a branch for a later year than the stored total replaces a grown total the branch has already calculated. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/simulation.py | 14 ++++- .../tests/test_stored_household_totals.py | 55 +++++++++++++++---- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index c402ec84f1..7f50bf8d62 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -750,8 +750,7 @@ def _stored_household_total(self, variable, period): # sum of person balances, which such data need not hold, so grow the # latest stored total instead, as Core grows an input with `uprating`. # Which years hold the total or its parts decides the result. - for name in (variable.name, *variable.adds): - self._record_derived(name) + self._record_derived(variable.name) source = self._latest_source(variable.name, period) if source is None: return None @@ -761,8 +760,17 @@ def _stored_household_total(self, variable, period): # Person balances stored for a later year than the total are # the newer observation, so their sum stands. return None + try: + index_name = stored_total_uprating(self.tax_benefit_system, variable.name) + except ValueError: + # A reform whose parts grow by different indices gives the total + # no index of its own, so its sum stands. + return None + # The grown total reads none of the parts, but a part input set later + # for a later year replaces it. + for part in variable.adds: + self._record_derived(part) stored = self.get_holder(variable.name).get_array(source, self.branch_name) - index_name = stored_total_uprating(self.tax_benefit_system, variable.name) return stored * self._index_growth(index_name, source, period) def _run_formula(self, variable, population, period): diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index 41e3a725c3..1d8dfcf547 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -39,7 +39,7 @@ _uprating_index_map, stored_total_uprating, ) -from policyengine_uk.model_api import YEAR, Household, Variable +from policyengine_uk.model_api import YEAR, Household, Person, Variable from policyengine_uk.scenarios import no_economic_assumptions pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -271,6 +271,25 @@ def test_a_total_set_on_a_branch_replaces_the_one_a_later_year_grew_from(): ) +def test_balances_set_on_a_branch_after_a_grown_total_replace_it(): + # A grown total reads none of the person balances, so a balance set on a + # branch for a later year than the stored total must still reach a year + # the branch has already calculated. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + branch = simulation.get_branch("balances") + between, later = PROJECTION_END_YEAR + 1, PROJECTION_END_YEAR + 3 + grown = branch.calculate("cash_isa", later)[0] + assert grown > 3_000 + branch.set_input("cash_isa_balance", between, np.array([600.0, 400.0])) + growth = _growth(simulation, "cash_isa_balance", later, base=between) + assert branch.calculate("cash_isa", later)[0] == pytest.approx( + 1_000 * growth, rel=1e-6 + ) + assert simulation.calculate("cash_isa", later)[0] == pytest.approx(grown, rel=1e-6) + + def test_person_balances_stored_after_the_total_are_summed_instead(): # A total and balances that a dataset stores for the same year: the total # takes precedence, and grows. Balances stored for later years than the @@ -370,24 +389,36 @@ def apply(self): @pytest.mark.parametrize( - "reform_adds", - ["stored_totals_test.parts", ["cash_isa_balance", "stored_totals_test.extra"]], - ids=["parameter", "list_with_a_parameter"], + "reform_adds, extra", + [ + ("stored_totals_test.parts", 0), + (["cash_isa_balance", "stored_totals_test.extra"], 100), + (["cash_isa_balance", "other_isa_holdings"], 0), + ], + ids=["parameter", "list_with_a_parameter", "parts_with_different_indices"], ) -def test_a_reform_sum_through_parameters_runs_in_years_with_no_stored_total( - reform_adds, +def test_a_reform_sum_with_no_index_for_the_total_runs_where_no_total_is_stored( + reform_adds, extra ): # Core lets `adds` name a parameter that lists the parts, or list a - # parameter beside variables. A reform that sums a total that way replaces - # its baseline sum, so Core runs it wherever no total is stored. + # parameter beside variables, and a reform can add a part that grows by + # another index (other_isa_holdings declares none). None of these gives + # the stored total an index to grow by, so Core runs the reform's sum + # wherever no total is stored, as it runs a reform formula. + class other_isa_holdings(Variable): + value_type = float + entity = Person + label = "other ISA holdings" + definition_period = YEAR + class cash_isa(Variable): value_type = float entity = Household - label = "cash ISA holdings, summed through parameters" + label = "cash ISA holdings, summed differently" definition_period = YEAR adds = reform_adds - class parameter_sum(Reform): + class other_sum(Reform): def apply(self): self.parameters.add_child( "stored_totals_test", @@ -399,17 +430,17 @@ def apply(self): }, ), ) + self.add_variable(other_isa_holdings) self.update_variable(cash_isa) simulation = Simulation( - reform=parameter_sum, + reform=other_sum, dataset=_dataset( [{"cash_isa": 5_000.0, "cash_isa_balance": 400.0, "share": 0.25}], person_columns=["cash_isa_balance"], household_columns=["cash_isa"], ), ) - extra = 0 if isinstance(reform_adds, str) else 100 later = PROJECTION_END_YEAR + 1 assert simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] > 5_000 balances = simulation.calculate("cash_isa_balance", later) From c31d62c7af78611d4e60d7228ecb709155aa04c2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 23:50:58 -0400 Subject: [PATCH 09/19] Say when a stored ISA total grows and when its parts' sum stands The variable documentation, the ISA and dataset-source pages and the changelog now say that the model grows the latest stored total after the last year the data hold, input frames and multi-year datasets included, and that balances held for a later year replace it. The changelog no longer counts stocks-and-shares ISA holdings that corporate_wealth already covers as lost capital, and the edited comments use the active voice. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2740.fixed.md | 2 +- .../isa-inputs-and-capital-flows.md | 17 +++++++------- docs/engineering/skills/dataset-sources.md | 22 ++++++++++--------- policyengine_uk/data/economic_assumptions.py | 9 ++++---- .../variables/household/wealth/cash_isa.py | 9 ++++---- .../wealth/household_lifetime_isa_balance.py | 9 ++++---- .../household/wealth/stocks_and_shares_isa.py | 11 +++++----- 7 files changed, 43 insertions(+), 36 deletions(-) diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md index 4eb6e9fcae..48d0bba474 100644 --- a/changelog.d/2740.fixed.md +++ b/changelog.d/2740.fixed.md @@ -1 +1 @@ -A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it in later years, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset stores those balances for a later year. Means-tested capital no longer drops by the ISA holdings from 2033. +A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after the last year it holds, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset holds those balances for a later year. An input frame or a multi-year dataset that stores a total now grows it the same way, rather than summing person balances it does not hold. Means-tested capital no longer drops from 2033 by the cash ISA holdings, or by stocks-and-shares ISA holdings that `corporate_wealth` does not already cover. diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index 3823214819..4990406176 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -100,14 +100,15 @@ established by the household tests. `cash_isa_balance`, and `stocks_and_shares_isa` sums their `stocks_and_shares_isa_balance`, as `household_lifetime_isa_balance` sums `lifetime_isa_balance`. A dataset or situation that stores a household total -keeps it: the total is recomputed only when it is not stored. The model -projects a total that a dataset stores with `savings`, `corporate_wealth` or -`lifetime_isa_balance` in every year, including years after the last one the -dataset projection reaches, unless the dataset stores the person balances for -a later year. A household input in a situation applies only to its own year; -the person balances carry the variable-level uprating to later years. The -means tests read the household totals, so a person's balance is pooled with -the household's other financial capital rather than attributed to their own +keeps it: the model sums the person balances only for a year with no stored +total. In each later year the model grows the latest total that a dataset +stores with `savings`, `corporate_wealth` or `lifetime_isa_balance` +respectively, including years after the dataset projection ends, unless the +dataset holds the person balances for a later year than the total. A +household input in a situation applies only to its own year; the person +balances carry the variable-level uprating to later years. The means tests +read the household totals, so they pool a person's balance with the +household's other financial capital rather than attribute it to their own benefit unit. Ordinary accessible cash ISA balances enter the capital sources for Universal diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index a60616d42e..2c72251223 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -157,16 +157,18 @@ Producers can export per-person `cash_isa_balance` and `stocks_and_shares_isa` sum them, but a stored household column takes precedence, so existing datasets that store only the household totals keep them in every year. `uprating_indices.yaml` projects them with `savings` and -`corporate_wealth` through the years the dataset projection covers. In later -years, and in any year a data source such as an input frame holds no total -for, the simulation grows the latest stored total by the index its person -balances declare (`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, -which also covers `household_lifetime_isa_balance`). Person balances stored -for a later year than the total replace it with their sum. Situations differ: -a household total given in one applies only to its own year, so a situation -reloaded from its input frame carries such a total forward. A producer that -exports the person balances should either stop exporting the household totals -or keep them equal to the sums; the model does not reconcile the two. +`corporate_wealth` through the years the dataset projection covers. After the +last year a data source holds a total for (the projection's last year, or the +last year of an input frame or a multi-year dataset), the simulation grows the +latest stored total by the index its person balances declare +(`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also covers +`household_lifetime_isa_balance`). Person balances that the data hold for a +later year than the total replace it with their sum. A situation differs: a +household total given in one applies only to its own year. An input frame is +data, so a situation reloaded from its input frame carries such a total +forward. A producer that exports the person balances should either stop +exporting the household totals or keep them equal to the sums; the model does +not reconcile the two. `cash_isa_subscriptions` and `stocks_and_shares_isa_subscriptions` are annual person flows, summed over all of a person's accounts of each type, that diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index a5c6e8cc31..58e53b1867 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -175,9 +175,9 @@ def growth_rate( # which falls to zero in data that store only the total, and Core does not # let a variable declare both `adds` and `uprating`. So in a simulation built # from data, Simulation grows the latest earlier stored total by the index -# its parts declare, as Core grows the parts. It sums the parts where no -# total is stored for that year or an earlier one, or where the parts are -# stored for a later year than the total. +# its parts declare, as Core grows the parts. It sums the parts where the +# data hold no total for that year or an earlier one, or hold the parts for +# a later year than the total. STORED_HOUSEHOLD_TOTALS = frozenset( {"cash_isa", "stocks_and_shares_isa", "household_lifetime_isa_balance"} ) @@ -211,7 +211,8 @@ def stored_total_uprating(tax_benefit_system, name: str) -> str | None: """The index that grows a stored household total: its parts' ``uprating``. Growing the total like its parts keeps a stored total equal to the sum it - replaces. ``None`` means the parts are held flat. + replaces. ``None`` holds the total flat, as Core holds parts with no + ``uprating``. Parts that declare different indices raise ``ValueError``. """ indices = { tax_benefit_system.variables[part].uprating diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index 6c1e5af29a..d85c3e085c 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -6,10 +6,11 @@ class cash_isa(Variable): documentation = ( "Amount held in cash Individual Savings Accounts: cash_isa_balance " "summed over the household's members. A dataset that stores this " - "column keeps the stored value, and projects it with savings in every " - "year: it is recomputed only when not stored, so a reform or situation " - "that edits cash_isa_balance leaves a stored total unchanged. A " - "household input in a situation applies only to its own year." + "column keeps the stored value, and the model grows the latest stored " + "value with savings in later years, so a reform that edits " + "cash_isa_balance leaves a stored total unchanged. Balances that the " + "data hold for a later year than the total replace it with their sum. " + "A household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py index 68d07a4234..409491184b 100644 --- a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py +++ b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py @@ -7,10 +7,11 @@ class household_lifetime_isa_balance(Variable): "Lifetime ISA balances summed over the household's members. Like " "lifetime_isa_balance, it is a component of gross_financial_wealth and " "must never be added to it. A dataset that stores this column keeps " - "the stored value, and projects it with lifetime_isa_balance in every " - "year: it is recomputed only when not stored, so a reform or situation " - "that edits lifetime_isa_balance leaves it unchanged. No means test " - "reads it." + "the stored value, and the model grows the latest stored value with " + "lifetime_isa_balance in later years, so a reform that edits " + "lifetime_isa_balance leaves it unchanged. Balances that the data hold " + "for a later year than the total replace it with their sum. No means " + "test reads it." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index 140420edd9..5bd9774058 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -7,11 +7,12 @@ class stocks_and_shares_isa(Variable): "Amount held in stocks and shares (investment) Individual Savings " "Accounts: stocks_and_shares_isa_balance summed over the household's " "members. Part of corporate_wealth, so the two must never be summed. " - "A dataset that stores this column keeps the stored value, and " - "projects it with corporate_wealth in every year: it is recomputed " - "only when not stored, so a reform or situation that edits " - "stocks_and_shares_isa_balance leaves a stored total unchanged. A " - "household input in a situation applies only to its own year." + "A dataset that stores this column keeps the stored value, and the " + "model grows the latest stored value with corporate_wealth in later " + "years, so a reform that edits stocks_and_shares_isa_balance leaves a " + "stored total unchanged. Balances that the data hold for a later year " + "than the total replace it with their sum. A household input in a " + "situation applies only to its own year." ) entity = Household definition_period = YEAR From 7373879d8e1919b4c295245c5bf5a6f39f10a280 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 00:14:24 -0400 Subject: [PATCH 10/19] Leave a reform sum that subtracts to Core, and tighten the ISA wording A reform that gave a stored ISA total `subtracts` took the grown-total path, which ignores what it subtracts. Simulation now grows a stored total only for a plain sum of variables, and a test covers the subtracting reform beside the parameter and different-index sums, and the ValueError that stored_total_uprating raises for parts that share no index. The variable documentation now says that a reform formula for the balances leaves a stored total unchanged, the docs say the model grows the latest earlier total in each year a source holds none for, the changelog covers frames and multi-year datasets that hold the balances too, and the edited test comments use the active voice. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2740.fixed.md | 2 +- .../isa-inputs-and-capital-flows.md | 7 +-- docs/engineering/skills/dataset-sources.md | 11 ++-- policyengine_uk/simulation.py | 11 ++-- .../tests/test_stored_household_totals.py | 52 +++++++++++++------ .../variables/household/wealth/cash_isa.py | 14 ++--- .../wealth/household_lifetime_isa_balance.py | 13 +++-- .../household/wealth/stocks_and_shares_isa.py | 17 +++--- 8 files changed, 73 insertions(+), 54 deletions(-) diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md index 48d0bba474..29b7d9065a 100644 --- a/changelog.d/2740.fixed.md +++ b/changelog.d/2740.fixed.md @@ -1 +1 @@ -A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after the last year it holds, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset holds those balances for a later year. An input frame or a multi-year dataset that stores a total now grows it the same way, rather than summing person balances it does not hold. Means-tested capital no longer drops from 2033 by the cash ISA holdings, or by stocks-and-shares ISA holdings that `corporate_wealth` does not already cover. +A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after the last year it holds, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset holds those balances for a later year. An input frame or a multi-year dataset that stores a total now grows it the same way after the years it holds it for, where it used to sum the person balances. Means-tested capital no longer drops from 2033 by the cash ISA holdings, or by stocks-and-shares ISA holdings that `corporate_wealth` does not already cover. diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index 4990406176..e7ac7cc4d9 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -102,9 +102,10 @@ established by the household tests. `lifetime_isa_balance`. A dataset or situation that stores a household total keeps it: the model sums the person balances only for a year with no stored total. In each later year the model grows the latest total that a dataset -stores with `savings`, `corporate_wealth` or `lifetime_isa_balance` -respectively, including years after the dataset projection ends, unless the -dataset holds the person balances for a later year than the total. A +stores by the index of `savings`, `corporate_wealth` or +`lifetime_isa_balance` respectively, including years after the dataset +projection ends, unless the dataset holds the person balances for a later +year than the total. A household input in a situation applies only to its own year; the person balances carry the variable-level uprating to later years. The means tests read the household totals, so they pool a person's balance with the diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 2c72251223..02f7ebc0b3 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -157,16 +157,15 @@ Producers can export per-person `cash_isa_balance` and `stocks_and_shares_isa` sum them, but a stored household column takes precedence, so existing datasets that store only the household totals keep them in every year. `uprating_indices.yaml` projects them with `savings` and -`corporate_wealth` through the years the dataset projection covers. After the -last year a data source holds a total for (the projection's last year, or the -last year of an input frame or a multi-year dataset), the simulation grows the -latest stored total by the index its person balances declare +`corporate_wealth` through the years the dataset projection covers. In each +year a data source holds no total for, the simulation grows the latest +earlier stored total by the index its person balances declare (`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also covers `household_lifetime_isa_balance`). Person balances that the data hold for a later year than the total replace it with their sum. A situation differs: a household total given in one applies only to its own year. An input frame is -data, so a situation reloaded from its input frame carries such a total -forward. A producer that exports the person balances should either stop +data, so a situation reloaded from its input frame grows such a total in +later years. A producer that exports the person balances should either stop exporting the household totals or keep them equal to the sums; the model does not reconcile the two. diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 7f50bf8d62..c9c5136923 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -766,20 +766,21 @@ def _stored_household_total(self, variable, period): # A reform whose parts grow by different indices gives the total # no index of its own, so its sum stands. return None - # The grown total reads none of the parts, but a part input set later - # for a later year replaces it. + # The grown total reads none of the parts, so record them: a part + # input that a branch sets later for a later year then replaces it. for part in variable.adds: self._record_derived(part) stored = self.get_holder(variable.name).get_array(source, self.branch_name) return stored * self._index_growth(index_name, source, period) def _run_formula(self, variable, population, period): - # Only a sum of listed variables: a reform that gives the variable a - # formula, sums a parameter or its list, or makes the variable a - # plain input, leaves it to Core. + # Only a sum of listed variables, with nothing subtracted: a reform + # that gives the variable a formula, sums a parameter or its list, + # subtracts, or makes the variable a plain input, leaves it to Core. if ( variable.name in STORED_HOUSEHOLD_TOTALS and variable.adds + and not variable.subtracts and not isinstance(variable.adds, str) and all(part in self.tax_benefit_system.variables for part in variable.adds) and variable.get_formula(period) is None diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index 1d8dfcf547..b75ee0ec01 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -18,6 +18,8 @@ the same amounts. """ +from types import SimpleNamespace + import numpy as np import pandas as pd import pytest @@ -47,8 +49,8 @@ DATA_YEAR = 2025 YEARS = range(DATA_YEAR, 2041) CAPITAL_YEARS = (DATA_YEAR, PROJECTION_END_YEAR, PROJECTION_END_YEAR + 1, YEARS[-1]) -# Each total, the person balance it sums, and the variable its documentation -# says it is projected with. +# Each total, the person balance it sums, and the variable whose index its +# documentation says the model grows it with. TOTALS = { "cash_isa": ("cash_isa_balance", "savings"), "stocks_and_shares_isa": ("stocks_and_shares_isa_balance", "corporate_wealth"), @@ -163,7 +165,7 @@ def test_stored_household_totals_grow_past_the_projection_horizon(): "scenario", [None, no_economic_assumptions], ids=["current_law", "no_growth"] ) def test_stored_totals_project_exactly_like_what_they_are_projected_with(scenario): - # cash_isa is stored equal to savings, stocks_and_shares_isa to + # The dataset stores cash_isa equal to savings, stocks_and_shares_isa to # corporate_wealth, and household_lifetime_isa_balance to the one # lifetime_isa_balance in the household: Core projects the second of each # pair, Simulation the first. Without economic growth both stay flat. @@ -363,7 +365,7 @@ def test_a_total_stored_in_an_input_frame_grows_like_savings(): def test_a_reform_formula_for_a_total_runs_in_years_with_no_stored_total(): # A reform that gives a total a formula replaces its sum, so Core runs the - # formula wherever no total is stored, as for any formula variable. + # formula in each year the data hold no total, as for any formula variable. class cash_isa(Variable): value_type = float entity = Household @@ -389,22 +391,29 @@ def apply(self): @pytest.mark.parametrize( - "reform_adds, extra", + "reform_adds, reform_subtracts, extra", [ - ("stored_totals_test.parts", 0), - (["cash_isa_balance", "stored_totals_test.extra"], 100), - (["cash_isa_balance", "other_isa_holdings"], 0), + ("stored_totals_test.parts", None, 0), + (["cash_isa_balance", "stored_totals_test.extra"], None, 100), + (["cash_isa_balance", "other_isa_holdings"], None, 0), + (["cash_isa_balance"], ["other_isa_holdings"], 0), + ], + ids=[ + "parameter", + "list_with_a_parameter", + "parts_with_different_indices", + "a_subtraction", ], - ids=["parameter", "list_with_a_parameter", "parts_with_different_indices"], ) -def test_a_reform_sum_with_no_index_for_the_total_runs_where_no_total_is_stored( - reform_adds, extra +def test_core_runs_any_other_reform_sum_where_the_data_hold_no_total( + reform_adds, reform_subtracts, extra ): - # Core lets `adds` name a parameter that lists the parts, or list a - # parameter beside variables, and a reform can add a part that grows by - # another index (other_isa_holdings declares none). None of these gives - # the stored total an index to grow by, so Core runs the reform's sum - # wherever no total is stored, as it runs a reform formula. + # Core lets `adds` name a parameter that lists the parts, list a + # parameter beside variables, or come with `subtracts`, and a reform can + # add a part that grows by another index (other_isa_holdings declares + # none). Simulation grows a stored total only for a plain sum of variables + # that share one index, so for any of these Core runs the reform's sum in + # each year the data hold no total, as it runs a reform formula. class other_isa_holdings(Variable): value_type = float entity = Person @@ -417,6 +426,7 @@ class cash_isa(Variable): label = "cash ISA holdings, summed differently" definition_period = YEAR adds = reform_adds + subtracts = reform_subtracts class other_sum(Reform): def apply(self): @@ -450,6 +460,16 @@ def apply(self): ) +def test_parts_with_different_indices_give_a_total_no_index(): + variables = { + "total": SimpleNamespace(adds=["first", "second"]), + "first": SimpleNamespace(uprating="gov.economic_assumptions.indices.a"), + "second": SimpleNamespace(uprating=None), + } + with pytest.raises(ValueError, match="different uprating indices"): + stored_total_uprating(SimpleNamespace(variables=variables), "total") + + def test_stored_totals_grow_by_the_index_they_are_projected_with(): from policyengine_uk.system import system diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index d85c3e085c..0214389cac 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -4,13 +4,13 @@ class cash_isa(Variable): label = "cash ISA holdings" documentation = ( - "Amount held in cash Individual Savings Accounts: cash_isa_balance " - "summed over the household's members. A dataset that stores this " - "column keeps the stored value, and the model grows the latest stored " - "value with savings in later years, so a reform that edits " - "cash_isa_balance leaves a stored total unchanged. Balances that the " - "data hold for a later year than the total replace it with their sum. " - "A household input in a situation applies only to its own year." + "Amount held in cash Individual Savings Accounts: cash_isa_balance summed " + "over the household's members. A dataset that stores this column keeps the " + "stored value, and in later years the model grows the latest stored value as " + "it grows savings, so a reform that gives cash_isa_balance a formula leaves a " + "stored total unchanged. Balances that the data hold for a later year than " + "the total replace it with their sum. A household input in a situation " + "applies only to its own year." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py index 409491184b..7987f66626 100644 --- a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py +++ b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py @@ -5,13 +5,12 @@ class household_lifetime_isa_balance(Variable): label = "household Lifetime ISA balance" documentation = ( "Lifetime ISA balances summed over the household's members. Like " - "lifetime_isa_balance, it is a component of gross_financial_wealth and " - "must never be added to it. A dataset that stores this column keeps " - "the stored value, and the model grows the latest stored value with " - "lifetime_isa_balance in later years, so a reform that edits " - "lifetime_isa_balance leaves it unchanged. Balances that the data hold " - "for a later year than the total replace it with their sum. No means " - "test reads it." + "lifetime_isa_balance, it is a component of gross_financial_wealth and must " + "never be added to it. A dataset that stores this column keeps the stored " + "value, and in later years the model grows the latest stored value as it " + "grows lifetime_isa_balance, so a reform that gives lifetime_isa_balance a " + "formula leaves it unchanged. Balances that the data hold for a later year " + "than the total replace it with their sum. No means test reads it." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index 5bd9774058..41e572e71d 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -4,15 +4,14 @@ class stocks_and_shares_isa(Variable): label = "stocks and shares ISA holdings" documentation = ( - "Amount held in stocks and shares (investment) Individual Savings " - "Accounts: stocks_and_shares_isa_balance summed over the household's " - "members. Part of corporate_wealth, so the two must never be summed. " - "A dataset that stores this column keeps the stored value, and the " - "model grows the latest stored value with corporate_wealth in later " - "years, so a reform that edits stocks_and_shares_isa_balance leaves a " - "stored total unchanged. Balances that the data hold for a later year " - "than the total replace it with their sum. A household input in a " - "situation applies only to its own year." + "Amount held in stocks and shares (investment) Individual Savings Accounts: " + "stocks_and_shares_isa_balance summed over the household's members. Part of " + "corporate_wealth, so the two must never be summed. A dataset that stores " + "this column keeps the stored value, and in later years the model grows the " + "latest stored value as it grows corporate_wealth, so a reform that gives " + "stocks_and_shares_isa_balance a formula leaves a stored total unchanged. " + "Balances that the data hold for a later year than the total replace it with " + "their sum. A household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR From 95cfa9d6b61a2a39993bfeecbf02c9894e6de35e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 03:15:40 -0400 Subject: [PATCH 11/19] Rewrap the ISA capital stocks paragraph Co-Authored-By: Claude Opus 5.5 --- .../assumptions/isa-inputs-and-capital-flows.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index e7ac7cc4d9..6a5dc3dc6e 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -102,15 +102,13 @@ established by the household tests. `lifetime_isa_balance`. A dataset or situation that stores a household total keeps it: the model sums the person balances only for a year with no stored total. In each later year the model grows the latest total that a dataset -stores by the index of `savings`, `corporate_wealth` or -`lifetime_isa_balance` respectively, including years after the dataset -projection ends, unless the dataset holds the person balances for a later -year than the total. A -household input in a situation applies only to its own year; the person -balances carry the variable-level uprating to later years. The means tests -read the household totals, so they pool a person's balance with the -household's other financial capital rather than attribute it to their own -benefit unit. +stores by the index of `savings`, `corporate_wealth` or `lifetime_isa_balance` +respectively, including years after the dataset projection ends, unless the +dataset holds the person balances for a later year than the total. A household +input in a situation applies only to its own year; the person balances carry +the variable-level uprating to later years. The means tests read the household +totals, so they pool a person's balance with the household's other financial +capital rather than attribute it to their own benefit unit. Ordinary accessible cash ISA balances enter the capital sources for Universal Credit, Housing Benefit, Pension Credit, Income Support, income-based JSA and From db6aa6c42ef6a3f5952a2b0b2da839e324bedbdb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:13:40 -0400 Subject: [PATCH 12/19] Grow a stored ISA total only while it stands for its stored parts STORED_HOUSEHOLD_TOTALS now maps each total to the person balances it sums. Simulation grows a stored total only while the variable sums exactly those parts, each once, with nothing subtracted and no formula, and while each part reads its inputs in that year: no formula, sum, subtraction, defined_for mask or neutralization. Any other reform gets Core's own calculation, as on main. Resets that follow an input change now delete the years of a stored total that the simulation calculated, through Core's own input records, and keep every year the data or a caller supplied. A total set for another year, or a part set for a later year, then reaches a year the default branch had already grown. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/data/economic_assumptions.py | 25 +- policyengine_uk/simulation.py | 52 +- .../tests/test_stored_household_totals.py | 602 +++++++++++++++--- policyengine_uk/utils/measurement_branches.py | 31 +- 4 files changed, 584 insertions(+), 126 deletions(-) diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 46d159c287..5984d1ce19 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -20,6 +20,7 @@ ) import hashlib +from types import MappingProxyType import yaml from policyengine_core.parameters import ParameterNode @@ -171,15 +172,21 @@ def growth_rate( PROJECTION_END_YEAR = 2032 # Household totals that sum person inputs (`adds`) but that a dataset can -# store itself (#2740). For a year with no stored total, Core runs the sum, -# which falls to zero in data that store only the total, and Core does not -# let a variable declare both `adds` and `uprating`. So in a simulation built -# from data, Simulation grows the latest earlier stored total by the index -# its parts declare, as Core grows the parts. It sums the parts where the -# data hold no total for that year or an earlier one, or hold the parts for -# a later year than the total. -STORED_HOUSEHOLD_TOTALS = frozenset( - {"cash_isa", "stocks_and_shares_isa", "household_lifetime_isa_balance"} +# store itself (#2740), each with the parts it stands for. For a year with no +# stored total, Core runs the sum, which falls to zero in data that store +# only the total, and Core does not let a variable declare both `adds` and +# `uprating`. So in a simulation built from data, Simulation grows the latest +# earlier stored total by the index its parts declare, as Core grows the +# parts, while the total still means what the data stored: a sum of exactly +# these parts, each a plain input. Core sums the parts where the variable +# differs, where the data hold no total for that year or an earlier one, or +# where they hold the parts for a later year than the total. +STORED_HOUSEHOLD_TOTALS = MappingProxyType( + { + "cash_isa": frozenset({"cash_isa_balance"}), + "stocks_and_shares_isa": frozenset({"stocks_and_shares_isa_balance"}), + "household_lifetime_isa_balance": frozenset({"lifetime_isa_balance"}), + } ) # Before this horizon was extended, Core uprated unmapped inputs from 2030. diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index c9c5136923..b653ca832c 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -85,6 +85,7 @@ from policyengine_uk.utils.data_source import built_from_data from policyengine_uk.utils.input_dataframe_export import input_dataframe from policyengine_uk.utils.measurement_branches import ( + drop_calculated_totals, drop_calculated_values, retained_input_variables, ) @@ -743,6 +744,36 @@ def _latest_source(self, name, period): source = self._isa_latest_input(name, period) return source + def _stands_for_stored_parts(self, variable, period) -> bool: + # A stored total means the sum of the parts STORED_HOUSEHOLD_TOTALS + # names for it, each a plain input. Grow it only while the variable + # still sums exactly those parts, each once, with nothing subtracted, + # and each part still reads its inputs: no formula, sum or subtraction + # for this period, no defined_for mask, not neutralized. Anything a + # reform changes there leaves the total to Core, as in data with no + # stored total. + parts = STORED_HOUSEHOLD_TOTALS.get(variable.name) + if ( + parts is None + or not built_from_data(self) + or variable.get_formula(period) is not None + or variable.subtracts + or sorted(variable.adds or ()) != sorted(parts) + ): + return False + for name in parts: + part = self.tax_benefit_system.variables.get(name) + if ( + part is None + or part.get_formula(period) is not None + or part.adds + or part.subtracts + or part.defined_for is not None + or part.is_neutralized + ): + return False + return True + def _stored_household_total(self, variable, period): # A dataset that stores a household total keeps it, projected like # its parts, in every year (#2740). Core serves the stored total for @@ -750,6 +781,8 @@ def _stored_household_total(self, variable, period): # sum of person balances, which such data need not hold, so grow the # latest stored total instead, as Core grows an input with `uprating`. # Which years hold the total or its parts decides the result. + if not self._stands_for_stored_parts(variable, period): + return None self._record_derived(variable.name) source = self._latest_source(variable.name, period) if source is None: @@ -763,8 +796,8 @@ def _stored_household_total(self, variable, period): try: index_name = stored_total_uprating(self.tax_benefit_system, variable.name) except ValueError: - # A reform whose parts grow by different indices gives the total - # no index of its own, so its sum stands. + # Parts that grow by different indices give the total no index of + # its own, so their sum stands. return None # The grown total reads none of the parts, so record them: a part # input that a branch sets later for a later year then replaces it. @@ -774,18 +807,7 @@ def _stored_household_total(self, variable, period): return stored * self._index_growth(index_name, source, period) def _run_formula(self, variable, population, period): - # Only a sum of listed variables, with nothing subtracted: a reform - # that gives the variable a formula, sums a parameter or its list, - # subtracts, or makes the variable a plain input, leaves it to Core. - if ( - variable.name in STORED_HOUSEHOLD_TOTALS - and variable.adds - and not variable.subtracts - and not isinstance(variable.adds, str) - and all(part in self.tax_benefit_system.variables for part in variable.adds) - and variable.get_formula(period) is None - and built_from_data(self) - ): + if variable.name in STORED_HOUSEHOLD_TOTALS: stored = self._stored_household_total(variable, period) if stored is not None: return stored @@ -1070,6 +1092,8 @@ def reset_calculations(self): for variable in self.tax_benefit_system.variables: if variable not in retained: self.delete_arrays(variable) + elif variable in STORED_HOUSEHOLD_TOTALS: + drop_calculated_totals(self, variable) def get_known_variables(self): variables = [] diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index b75ee0ec01..ebdca3f0f3 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -4,10 +4,12 @@ balances, but a dataset can store the household total instead. Up to PROJECTION_END_YEAR the dataset projection grows a stored total; after it, Simulation grows the latest stored total by the index of its parts, the index -each variable's documentation projects it with. A dataset that stores only the -person balances, or stores them for a later year than the total, gets their -sum, and so does a situation for a year it gives no total for. An input frame -is data, so a total it stores grows too. +each variable's documentation projects it with, while the variable still sums +the parts STORED_HOUSEHOLD_TOTALS names and each part reads its inputs. A +dataset that stores only the person balances, or stores them for a later year +than the total, gets their sum, and so does a situation for a year it gives no +total for, and a reform that changes the sum or a part. An input frame is +data, so a total it stores grows too. Invariants, for any non-negative amounts: @@ -15,7 +17,12 @@ positive, and from PROJECTION_END_YEAR grows by its declared index; - with person balances stored instead, the total equals their sum every year; - the two forms give the same totals and means-test capital when they hold - the same amounts. + the same amounts; +- after any inputs and calculations, a total keeps every value the data or a + caller supplied, and equals what a fresh calculation gives in other years; +- a grown total grows as Core grows an input with its parts' index; +- a reform that changes the sum or a part gets Core's sum, and a reform that + declares the same variables again changes nothing. """ from types import SimpleNamespace @@ -28,6 +35,7 @@ from policyengine_core.parameters import ParameterNode from policyengine_core.periods import period from policyengine_core.reforms import Reform +from policyengine_core.simulations import Simulation as CoreSimulation from policyengine_core.simulations.simulation import _uprating_index_value from policyengine_uk import Simulation @@ -65,6 +73,7 @@ "esa_income_assessable_capital", "council_tax_reduction_assessable_capital", ) +CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" PROPERTY_SETTINGS = settings( max_examples=5, deadline=None, @@ -120,16 +129,61 @@ def _values(simulation, name, years=YEARS): return {year: simulation.calculate(name, year).astype(float) for year in years} -def _growth(simulation, name, year, base=PROJECTION_END_YEAR): - """Growth from ``base`` to ``year`` by the index ``name`` declares.""" - index = simulation.tax_benefit_system.parameters.get_child( - simulation.tax_benefit_system.variables[name].uprating - ) +def _index_growth(simulation, index_name, year, base): + """Growth from ``base`` to ``year`` by the index ``index_name``.""" + index = simulation.tax_benefit_system.parameters.get_child(index_name) return _uprating_index_value(index, period(year).start) / _uprating_index_value( index, period(base).start ) +def _growth(simulation, name, year, base=PROJECTION_END_YEAR): + """Growth from ``base`` to ``year`` by the index ``name`` declares.""" + index_name = simulation.tax_benefit_system.variables[name].uprating + return _index_growth(simulation, index_name, year, base) + + +def _core_sum(simulation, name, year): + """What Core calculates for ``name`` in ``year`` from its definition.""" + variable = simulation.tax_benefit_system.variables[name] + population = simulation.get_variable_population(name) + return CoreSimulation._run_formula(simulation, variable, population, period(year)) + + +def _reform(*changes): + """A reform that applies each change to the tax-benefit system.""" + + class reform(Reform): + def apply(self): + for change in changes: + change(self) + + return reform + + +def _redeclared(name, **attributes): + """``name`` as a reform declares it in full, with ``attributes``.""" + from policyengine_uk.system import system + + baseline = system.variables[name] + return type( + name, + (Variable,), + { + "value_type": baseline.value_type, + "entity": Person if baseline.entity.key == "person" else Household, + "definition_period": baseline.definition_period, + "label": baseline.label, + **attributes, + }, + ) + + +def _updated(name, **attributes): + """``name`` as a reform updates it, keeping every other attribute.""" + return type(name, (Variable,), attributes) + + def test_stored_household_totals_grow_past_the_projection_horizon(): simulation = Simulation( dataset=_dataset( @@ -203,6 +257,45 @@ def test_stored_totals_project_exactly_like_what_they_are_projected_with(scenari assert share_like[0] == simulation.calculate("corporate_wealth", year)[0] +@pytest.mark.parametrize( + "index", [None, CPI, "parts"], ids=["no_index", "cpi", "parts_index"] +) +def test_a_grown_total_grows_as_core_grows_its_part(index): + # The data store a total and the balance it sums for the same year, so + # the total takes precedence and grows by the balance's index, while Core + # grows the balance itself. A reform can give the balance another index, + # or none, when Core carries the balance flat and the total stays flat. + part = "cash_isa_balance" + reform = None + if index != "parts": + reform = _reform( + lambda system: system.replace_variable( + _redeclared(part, uprating=index, default_value=0) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 1_000.0, part: 1_000.0, "share": 0.3}], + person_columns=[part], + household_columns=["cash_isa"], + ), + ) + if index != "parts": + assert simulation.tax_benefit_system.variables[part].uprating == index + for year in YEARS: + balances = simulation.calculate(part, year) + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ), year + later = simulation.calculate("cash_isa", YEARS[-1])[0] + horizon = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + if index is None: + assert later == horizon + else: + assert later > horizon + + def test_person_balances_still_sum_without_a_stored_total(): simulation = Simulation( dataset=_dataset( @@ -226,6 +319,29 @@ def test_person_balances_still_sum_without_a_stored_total(): ) +def test_a_stored_zero_total_takes_precedence_over_same_year_balances(): + # A total of zero is an observation, as in the years the data hold: it + # stays zero beside positive balances stored for the same year, in a + # household on its own and beside one with a positive total. + zero = {"cash_isa": 0.0, "cash_isa_balance": 1_000.0, "share": 0.5} + positive = {"cash_isa": 2_000.0, "cash_isa_balance": 500.0, "share": 0.5} + for households in ([zero], [zero, positive]): + simulation = Simulation( + dataset=_dataset( + households, + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ) + ) + for year in CAPITAL_YEARS: + totals = simulation.calculate("cash_isa", year) + assert totals[0] == 0, year + assert simulation.calculate("cash_isa_balance", year)[:2].sum() > 0 + if len(households) == 2: + savings = simulation.calculate("savings", year)[1] + assert totals[1] == pytest.approx(2 * savings, rel=1e-6), year + + def test_a_situation_total_is_used_only_for_the_year_it_is_given_for(): simulation = Simulation( situation={ @@ -273,23 +389,105 @@ def test_a_total_set_on_a_branch_replaces_the_one_a_later_year_grew_from(): ) -def test_balances_set_on_a_branch_after_a_grown_total_replace_it(): - # A grown total reads none of the person balances, so a balance set on a - # branch for a later year than the stored total must still reach a year - # the branch has already calculated. +def _one_person_frame(**columns): + return pd.DataFrame( + { + "person_id__2025": [1], + "person_benunit_id__2025": [1], + "person_household_id__2025": [1], + **{name: [amount] for name, amount in columns.items()}, + } + ) + + +@pytest.mark.parametrize("branch", [None, "edited"], ids=["default", "named"]) +def test_a_total_set_for_another_year_replaces_a_grown_total_already_calculated( + branch, +): + # The data supply a total for 2025 only. A simulation that has grown it + # to 2035, then gets a total for 2030, grows that one instead; and one + # that has grown the 2030 total to 2037, then gets a total for 2036, + # grows that. The supplied totals keep their values. + simulation = Simulation(dataset=_one_person_frame(cash_isa__2025=3_000.0)) + target = simulation if branch is None else simulation.get_branch(branch) + assert target.calculate("cash_isa", 2035)[0] == pytest.approx( + 3_000 * _growth(simulation, "savings", 2035, base=2025), rel=1e-6 + ) + target.set_input("cash_isa", 2030, np.array([10_000.0])) + assert target.calculate("cash_isa", 2035)[0] == pytest.approx( + 10_000 * _growth(simulation, "savings", 2035, base=2030), rel=1e-6 + ) + assert target.calculate("cash_isa", 2037)[0] == pytest.approx( + 10_000 * _growth(simulation, "savings", 2037, base=2030), rel=1e-6 + ) + target.set_input("cash_isa", 2036, np.array([20_000.0])) + assert target.calculate("cash_isa", 2037)[0] == pytest.approx( + 20_000 * _growth(simulation, "savings", 2037, base=2036), rel=1e-6 + ) + for year, amount in ((2025, 3_000), (2030, 10_000), (2036, 20_000)): + assert target.calculate("cash_isa", year)[0] == amount + + +@pytest.mark.parametrize("branch", [None, "balances"], ids=["default", "named"]) +def test_balances_set_after_a_grown_total_replace_it(branch): + # A grown total reads none of the person balances, so a balance set for a + # later year than the stored total must still reach a year the + # simulation has already calculated. simulation = Simulation( dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) ) - branch = simulation.get_branch("balances") + target = simulation if branch is None else simulation.get_branch(branch) between, later = PROJECTION_END_YEAR + 1, PROJECTION_END_YEAR + 3 - grown = branch.calculate("cash_isa", later)[0] + grown = target.calculate("cash_isa", later)[0] assert grown > 3_000 - branch.set_input("cash_isa_balance", between, np.array([600.0, 400.0])) + target.set_input("cash_isa_balance", between, np.array([600.0, 400.0])) growth = _growth(simulation, "cash_isa_balance", later, base=between) - assert branch.calculate("cash_isa", later)[0] == pytest.approx( + assert target.calculate("cash_isa", later)[0] == pytest.approx( 1_000 * growth, rel=1e-6 ) - assert simulation.calculate("cash_isa", later)[0] == pytest.approx(grown, rel=1e-6) + if branch is not None: + assert simulation.calculate("cash_isa", later)[0] == pytest.approx( + grown, rel=1e-6 + ) + + +def test_a_grown_total_records_what_it_depends_on(): + # Inside a calculation, an input set on a branch reaches the values the + # branch holds only through Core's records of what each depends on. A + # grown total depends on the total and on its parts in every other year. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + branch = simulation.get_branch("recorded") + later = PROJECTION_END_YEAR + 3 + branch.calculate("cash_isa", later) + grown = branch.get_holder("cash_isa")._stored_sequence_number( + period(later), "recorded" + ) + history = branch._get_store_history() + for name in ("cash_isa", "cash_isa_balance"): + since = history.earliest_dependency(name, period(later - 1)) + assert since is not None and since <= grown, name + + +def test_resets_drop_grown_totals_and_keep_supplied_ones(): + # A stored total counts as an input by name, but the simulation + # calculates it for the years the data do not hold. Resetting the + # calculations deletes those years and keeps every supplied one. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + holder = simulation.get_holder("cash_isa") + supplied = {year: holder.get_array(period(year)) for year in YEARS} + assert all(supplied[year] is not None for year in range(DATA_YEAR, 2033)) + later = PROJECTION_END_YEAR + 3 + simulation.calculate("cash_isa", later) + assert holder.get_array(period(later)) is not None + simulation.reset_calculations() + assert holder.get_array(period(later)) is None + for year in range(DATA_YEAR, PROJECTION_END_YEAR + 1): + assert holder.get_array(period(year)) == pytest.approx(supplied[year]) + assert holder.get_array(period(DATA_YEAR))[0] == 3_000 def test_person_balances_stored_after_the_total_are_summed_instead(): @@ -363,100 +561,250 @@ def test_a_total_stored_in_an_input_frame_grows_like_savings(): ) -def test_a_reform_formula_for_a_total_runs_in_years_with_no_stored_total(): - # A reform that gives a total a formula replaces its sum, so Core runs the - # formula in each year the data hold no total, as for any formula variable. - class cash_isa(Variable): - value_type = float - entity = Household - label = "cash ISA holdings, twice savings" - definition_period = YEAR +def _twice_savings(household, period, parameters): + return 2 * household("savings", period) - def formula(household, period, parameters): - return 2 * household("savings", period) - class twice_savings(Reform): - def apply(self): - self.update_variable(cash_isa) +def _a_fixed_balance(person, period, parameters): + return person.filled_array(123.0) - simulation = Simulation( - reform=twice_savings, - dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]), + +def _add_parameters(system): + system.parameters.add_child( + "stored_totals_test", + ParameterNode( + "stored_totals_test", + data={ + "parts": {"values": {"2000-01-01": ["cash_isa_balance"]}}, + "extra": {"values": {"2000-01-01": 100}}, + }, + ), ) - for year, multiple in ((PROJECTION_END_YEAR, 3), (PROJECTION_END_YEAR + 1, 2)): - savings = simulation.calculate("savings", year)[0] - assert simulation.calculate("cash_isa", year)[0] == pytest.approx( - multiple * savings, rel=1e-6 - ) + + +class other_isa_holdings(Variable): + value_type = float + entity = Person + label = "other ISA holdings" + definition_period = YEAR + + +# Changes to what cash_isa sums, and to stocks_and_shares_isa_balance, the +# part stocks_and_shares_isa sums. Each pair runs in one simulation. +SUM_CHANGES = { + "a_formula": lambda system: system.update_variable( + _updated("cash_isa", formula=_twice_savings) + ), + "a_parameter_list": lambda system: system.update_variable( + _updated("cash_isa", adds="stored_totals_test.parts") + ), + "a_parameter_among_the_parts": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "stored_totals_test.extra"]) + ), + "a_part_with_no_index": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "other_isa_holdings"]) + ), + "a_subtraction": lambda system: system.update_variable( + _updated("cash_isa", subtracts=["other_isa_holdings"]) + ), + "an_extra_part": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "stocks_and_shares_isa_balance"]) + ), + "the_part_twice": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "cash_isa_balance"]) + ), + "another_part": lambda system: system.update_variable( + _updated("cash_isa", adds=["stocks_and_shares_isa_balance"]) + ), +} +PART = "stocks_and_shares_isa_balance" +PART_CHANGES = { + "a_formula": lambda system: system.replace_variable( + _redeclared(PART, formula=_a_fixed_balance) + ), + "a_sum": lambda system: system.replace_variable( + _redeclared(PART, adds=["cash_isa_balance"]) + ), + "a_subtraction": lambda system: system.replace_variable( + _redeclared(PART, subtracts=["cash_isa_balance"]) + ), + "a_mask": lambda system: system.update_variable( + _updated(PART, defined_for="is_child") + ), + "neutralization": lambda system: system.neutralize_variable(PART), +} +CHANGE_PAIRS = [ + (sum_change, list(PART_CHANGES)[index % len(PART_CHANGES)]) + for index, sum_change in enumerate(SUM_CHANGES) +] @pytest.mark.parametrize( - "reform_adds, reform_subtracts, extra", - [ - ("stored_totals_test.parts", None, 0), - (["cash_isa_balance", "stored_totals_test.extra"], None, 100), - (["cash_isa_balance", "other_isa_holdings"], None, 0), - (["cash_isa_balance"], ["other_isa_holdings"], 0), - ], - ids=[ - "parameter", - "list_with_a_parameter", - "parts_with_different_indices", - "a_subtraction", - ], + "sum_change, part_change", + CHANGE_PAIRS, + ids=[f"sum_{total}-part_{part}" for total, part in CHANGE_PAIRS], ) -def test_core_runs_any_other_reform_sum_where_the_data_hold_no_total( - reform_adds, reform_subtracts, extra -): - # Core lets `adds` name a parameter that lists the parts, list a - # parameter beside variables, or come with `subtracts`, and a reform can - # add a part that grows by another index (other_isa_holdings declares - # none). Simulation grows a stored total only for a plain sum of variables - # that share one index, so for any of these Core runs the reform's sum in - # each year the data hold no total, as it runs a reform formula. - class other_isa_holdings(Variable): - value_type = float - entity = Person - label = "other ISA holdings" - definition_period = YEAR - - class cash_isa(Variable): - value_type = float - entity = Household - label = "cash ISA holdings, summed differently" - definition_period = YEAR - adds = reform_adds - subtracts = reform_subtracts - - class other_sum(Reform): - def apply(self): - self.parameters.add_child( - "stored_totals_test", - ParameterNode( - "stored_totals_test", - data={ - "parts": {"values": {"2000-01-01": ["cash_isa_balance"]}}, - "extra": {"values": {"2000-01-01": 100}}, - }, - ), +def test_core_sums_a_total_whose_sum_or_part_a_reform_changes(sum_change, part_change): + # A stored total stands for the sum of exactly the parts + # STORED_HOUSEHOLD_TOTALS names, each a plain input. A reform that sums + # anything else, or that calculates, masks or neutralizes a part, gets + # Core's own calculation in each year the data hold no total, as on data + # that store no total. Years the data hold keep the stored total. + reform = _reform( + _add_parameters, + lambda system: system.add_variable(other_isa_holdings), + SUM_CHANGES[sum_change], + PART_CHANGES[part_change], + ) + household = { + "cash_isa": 5_000.0, + "stocks_and_shares_isa": 7_000.0, + "cash_isa_balance": 400.0, + "stocks_and_shares_isa_balance": 1_000.0, + "share": 0.25, + } + simulation = Simulation( + reform=reform, + dataset=_dataset( + [household], + person_columns=["cash_isa_balance", "stocks_and_shares_isa_balance"], + household_columns=["cash_isa", "stocks_and_shares_isa"], + ), + ) + for name in ("cash_isa", "stocks_and_shares_isa"): + stored = simulation.calculate(name, PROJECTION_END_YEAR)[0] + assert stored > household[name] + assert stored == simulation.baseline.calculate(name, PROJECTION_END_YEAR)[0] + for year in (PROJECTION_END_YEAR + 1, YEARS[-1]): + total = simulation.calculate(name, year)[0] + assert total == pytest.approx( + _core_sum(simulation, name, year)[0], rel=1e-6, abs=1e-6 + ), (name, year) + grown = stored * _growth(simulation, "savings", year) + assert total != pytest.approx(grown, rel=1e-3), (name, year) + + +def test_a_part_formula_takes_over_from_the_year_it_starts(): + # Until its formula starts, the part reads its inputs, so the stored + # total stands; a part with no index holds it flat, as Core holds the + # part. From the formula's first year, Core sums the parts. + reform = _reform( + lambda system: system.replace_variable( + _redeclared( + "cash_isa_balance", + default_value=0, + formula_2034=_a_fixed_balance, ) - self.add_variable(other_isa_holdings) - self.update_variable(cash_isa) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]), + ) + horizon = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert horizon > 3_000 + assert simulation.calculate("cash_isa", 2033)[0] == horizon + for year in (2034, YEARS[-1]): + assert simulation.calculate("cash_isa", year)[0] == pytest.approx(246) + + +@pytest.mark.parametrize("method", ["replace", "update"]) +def test_reforms_that_declare_a_total_and_its_part_again_change_nothing(method): + from policyengine_uk.system import system as baseline_system + + def declare_again(system): + for total, parts in STORED_HOUSEHOLD_TOTALS.items(): + for name in (total, *parts): + if method == "replace": + system.replace_variable(type(baseline_system.variables[name])) + else: + system.update_variable(_updated(name)) + households = [ + { + "cash_isa": 3_000.0, + "stocks_and_shares_isa": 2_000.0, + "household_lifetime_isa_balance": 900.0, + "lifetime_isa_balance": 900.0, + "share": 0.5, + }, + { + "cash_isa": 0.0, + "stocks_and_shares_isa": 8_000.0, + "household_lifetime_isa_balance": 0.0, + "lifetime_isa_balance": 0.0, + "share": 1.0, + }, + ] + data = dict( + households=households, + person_columns=["lifetime_isa_balance"], + household_columns=TOTALS, + ) + baseline = Simulation(dataset=_dataset(**data)) + reformed = Simulation(reform=_reform(declare_again), dataset=_dataset(**data)) + for name in TOTALS: + for year in YEARS: + assert ( + reformed.calculate(name, year) == baseline.calculate(name, year) + ).all(), (name, year) + for year in CAPITAL_YEARS: + assert ( + reformed.calculate("uc_assessable_capital", year) + == baseline.calculate("uc_assessable_capital", year) + ).all(), year + + +class other_gdp_holdings(Variable): + value_type = float + entity = Person + label = "other holdings that grow with GDP per capita" + definition_period = YEAR + uprating = "gov.economic_assumptions.indices.obr.per_capita.gdp" + + +def test_a_total_of_several_parts_grows_only_by_an_index_they_share(monkeypatch): + # A total can stand for several parts. It grows by their index when they + # share one; parts that grow by different indices give it none, so Core + # sums them. + import policyengine_uk.simulation as uk_simulation + + parts = { + "cash_isa": ["cash_isa_balance", "other_isa_holdings"], + "stocks_and_shares_isa": [ + "stocks_and_shares_isa_balance", + "other_gdp_holdings", + ], + } + monkeypatch.setattr( + uk_simulation, + "STORED_HOUSEHOLD_TOTALS", + { + **STORED_HOUSEHOLD_TOTALS, + **{name: frozenset(names) for name, names in parts.items()}, + }, + ) + reform = _reform( + lambda system: system.add_variable(other_isa_holdings), + lambda system: system.add_variable(other_gdp_holdings), + *( + lambda system, name=name, names=names: system.update_variable( + _updated(name, adds=names) + ) + for name, names in parts.items() + ), + ) simulation = Simulation( - reform=other_sum, + reform=reform, dataset=_dataset( - [{"cash_isa": 5_000.0, "cash_isa_balance": 400.0, "share": 0.25}], - person_columns=["cash_isa_balance"], - household_columns=["cash_isa"], + [{"cash_isa": 5_000.0, "stocks_and_shares_isa": 7_000.0}], + household_columns=parts, ), ) later = PROJECTION_END_YEAR + 1 - assert simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] > 5_000 - balances = simulation.calculate("cash_isa_balance", later) - assert balances.sum() > 400 - assert simulation.calculate("cash_isa", later)[0] == pytest.approx( - balances.sum() + extra, rel=1e-6 + assert simulation.calculate("cash_isa", later)[0] == 0 + assert simulation.calculate("stocks_and_shares_isa", later)[0] == pytest.approx( + 7_000 * _growth(simulation, "savings", later, base=DATA_YEAR), rel=1e-6 ) @@ -470,11 +818,24 @@ def test_parts_with_different_indices_give_a_total_no_index(): stored_total_uprating(SimpleNamespace(variables=variables), "total") +def test_stored_totals_name_the_parts_their_variables_sum(): + # A stored total grows only while its variable sums exactly these parts, + # so the two must not drift apart. + from policyengine_uk.system import system + + for name, parts in STORED_HOUSEHOLD_TOTALS.items(): + adds = system.variables[name].adds + assert isinstance(adds, list) and len(adds) == len(parts), name + assert set(adds) == parts, name + for part in parts: + assert system.variables[part].is_input_variable(), part + + def test_stored_totals_grow_by_the_index_they_are_projected_with(): from policyengine_uk.system import system for name, (part, projected_with) in TOTALS.items(): - assert system.variables[name].adds == [part] + assert STORED_HOUSEHOLD_TOTALS[name] == {part} index = system.variables[projected_with].uprating assert index is not None assert stored_total_uprating(system, name) == index @@ -497,7 +858,7 @@ def test_every_projected_household_sum_of_uprated_inputs_is_a_stored_total(): for part in system.variables[name].adds ) } - assert sums_of_uprated_inputs == STORED_HOUSEHOLD_TOTALS + assert sums_of_uprated_inputs == set(STORED_HOUSEHOLD_TOTALS) AMOUNTS = st.integers(min_value=0, max_value=200_000_000).map(lambda p: p / 100) @@ -560,3 +921,44 @@ def test_stored_totals_and_person_balances_agree_in_every_year(households): assert totals_only.calculate(name, year) == pytest.approx( balances_only.calculate(name, year), rel=1e-5, abs=1e-3 ), (name, year) + + +INPUT_YEARS = st.integers(min_value=DATA_YEAR + 1, max_value=YEARS[-1]) +STEPS = st.lists( + st.one_of( + st.tuples(st.just("calculate"), INPUT_YEARS), + st.tuples(st.just("cash_isa"), INPUT_YEARS, AMOUNTS), + st.tuples(st.just("cash_isa_balance"), INPUT_YEARS, AMOUNTS), + ), + min_size=1, + max_size=8, +) + + +@PROPERTY_SETTINGS +@given(steps=STEPS, on_branch=st.booleans()) +def test_a_total_after_any_inputs_equals_a_fresh_calculation(steps, on_branch): + # Totals or balances set for any year, between calculations of any year: + # each calculated total equals what the same simulation gives with every + # calculated value dropped, and every supplied total keeps its value. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + target = simulation.get_branch("edited") if on_branch else simulation + supplied = { + year: target.calculate("cash_isa", year)[0] for year in range(DATA_YEAR, 2033) + } + for step in [*steps, ("calculate", YEARS[-1])]: + name, year = step[:2] + if name == "calculate": + fresh = target.clone() + fresh.drop_computed_arrays() + calculated = target.calculate("cash_isa", year)[0] + assert calculated == fresh.calculate("cash_isa", year)[0], (step, steps) + elif name == "cash_isa": + target.set_input(name, year, np.array([step[2]])) + supplied[year] = np.float32(step[2]) + else: + target.set_input(name, year, np.array([step[2], 0.0])) + for year, amount in supplied.items(): + assert target.calculate("cash_isa", year)[0] == amount, (year, steps) diff --git a/policyengine_uk/utils/measurement_branches.py b/policyengine_uk/utils/measurement_branches.py index 672eca5eac..9de444cd95 100644 --- a/policyengine_uk/utils/measurement_branches.py +++ b/policyengine_uk/utils/measurement_branches.py @@ -42,14 +42,35 @@ def retained_input_variables(simulation) -> set: return retained +def drop_calculated_totals(simulation, name: str) -> None: + """Delete the values of a stored household total that the simulation + calculated, keeping those the data or a caller supplied. + + A total in STORED_HOUSEHOLD_TOTALS that the data store counts as an input + by name, but the simulation calculates it for the years the data do not + hold, from the stored years and the parts. Core's own record of inputs + (``_user_input_keys``, checked by ``Holder._stores_user_input``) decides + which values go, as it does for ``drop_computed_arrays``. + """ + simulation.get_holder(name)._drop_computed() + cache = getattr(simulation, "_fast_cache", None) + if cache: + simulation._fast_cache = { + key: value for key, value in cache.items() if key[0] != name + } + + def drop_calculated_values(simulation) -> None: """Delete the values a simulation has calculated, keeping its inputs. Inputs are variables without a formula, variables the simulation was built from, and variables set since through set_input, as a scenario's - simulation modifier does to formula variables. Only variables the - simulation holds values for are visited. + simulation modifier does to formula variables. A stored household total + keeps only the years supplied for it (see drop_calculated_totals). Only + variables the simulation holds values for are visited. """ + from policyengine_uk.data.economic_assumptions import STORED_HOUSEHOLD_TOTALS + supplied = retained_input_variables(simulation) variables = simulation.tax_benefit_system.variables held = { @@ -59,7 +80,11 @@ def drop_calculated_values(simulation) -> None: } for name in held: variable = variables.get(name) - if variable is None or name in supplied or variable.is_input_variable(): + if variable is None: + continue + if name in supplied or variable.is_input_variable(): + if name in STORED_HOUSEHOLD_TOTALS: + drop_calculated_totals(simulation, name) continue simulation.delete_arrays(name) From f6850ab80284f29982cd14667a8c68b7eb2136f2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:13:43 -0400 Subject: [PATCH 13/19] Say when a stored ISA total grows and when a reform gets the sum Co-Authored-By: Claude Opus 5.5 --- changelog.d/2740.fixed.md | 2 +- docs/book/assumptions/isa-inputs-and-capital-flows.md | 11 ++++++----- docs/engineering/skills/dataset-sources.md | 6 ++++-- .../variables/household/wealth/cash_isa.py | 10 +++++----- .../wealth/household_lifetime_isa_balance.py | 9 +++++---- .../household/wealth/stocks_and_shares_isa.py | 11 ++++++----- 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md index 29b7d9065a..cd5fc9bd0a 100644 --- a/changelog.d/2740.fixed.md +++ b/changelog.d/2740.fixed.md @@ -1 +1 @@ -A dataset that stores a household `cash_isa`, `stocks_and_shares_isa` or `household_lifetime_isa_balance` total keeps it after the last year it holds, those after 2032 included, grown by the index of the person balances it stands for, unless the dataset holds those balances for a later year. An input frame or a multi-year dataset that stores a total now grows it the same way after the years it holds it for, where it used to sum the person balances. Means-tested capital no longer drops from 2033 by the cash ISA holdings, or by stocks-and-shares ISA holdings that `corporate_wealth` does not already cover. +Keep the household ISA totals a dataset stores after the years it holds, grown like the person balances they sum, so means-tested capital no longer drops by those holdings from 2033. diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index 6a5dc3dc6e..ce1d7c2c18 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -104,11 +104,12 @@ keeps it: the model sums the person balances only for a year with no stored total. In each later year the model grows the latest total that a dataset stores by the index of `savings`, `corporate_wealth` or `lifetime_isa_balance` respectively, including years after the dataset projection ends, unless the -dataset holds the person balances for a later year than the total. A household -input in a situation applies only to its own year; the person balances carry -the variable-level uprating to later years. The means tests read the household -totals, so they pool a person's balance with the household's other financial -capital rather than attribute it to their own benefit unit. +dataset holds the person balances for a later year than the total, or a reform +changes what the total sums or stops a person balance reading its inputs. A +household input in a situation applies only to its own year; the person +balances carry the variable-level uprating to later years. The means tests read +the household totals, so they pool a person's balance with the household's +other financial capital rather than attribute it to their own benefit unit. Ordinary accessible cash ISA balances enter the capital sources for Universal Credit, Housing Benefit, Pension Credit, Income Support, income-based JSA and diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 9bb593175f..f9d7a2419d 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -161,8 +161,10 @@ them in every year. `uprating_indices.yaml` projects them with `savings` and year a data source holds no total for, the simulation grows the latest earlier stored total by the index its person balances declare (`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also covers -`household_lifetime_isa_balance`). Person balances that the data hold for a -later year than the total replace it with their sum. A situation differs: a +`household_lifetime_isa_balance`, with the person balance each total sums). +Person balances that the data hold for a later year than the total replace it +with their sum, and so does any reform that changes what a total sums or +stops one of its person balances reading its inputs. A situation differs: a household total given in one applies only to its own year. An input frame is data, so a situation reloaded from its input frame grows such a total in later years. A producer that exports the person balances should either stop diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index 0214389cac..123d9eec6a 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -6,11 +6,11 @@ class cash_isa(Variable): documentation = ( "Amount held in cash Individual Savings Accounts: cash_isa_balance summed " "over the household's members. A dataset that stores this column keeps the " - "stored value, and in later years the model grows the latest stored value as " - "it grows savings, so a reform that gives cash_isa_balance a formula leaves a " - "stored total unchanged. Balances that the data hold for a later year than " - "the total replace it with their sum. A household input in a situation " - "applies only to its own year." + "stored value, and the model grows the latest stored value into later years " + "as it grows savings. In a later year the model sums the balances instead " + "where the data hold them for a later year than the total, or where a reform " + "changes what cash_isa sums or stops cash_isa_balance reading its inputs. A " + "household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py index 7987f66626..44bfbcd872 100644 --- a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py +++ b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py @@ -7,10 +7,11 @@ class household_lifetime_isa_balance(Variable): "Lifetime ISA balances summed over the household's members. Like " "lifetime_isa_balance, it is a component of gross_financial_wealth and must " "never be added to it. A dataset that stores this column keeps the stored " - "value, and in later years the model grows the latest stored value as it " - "grows lifetime_isa_balance, so a reform that gives lifetime_isa_balance a " - "formula leaves it unchanged. Balances that the data hold for a later year " - "than the total replace it with their sum. No means test reads it." + "value, and the model grows the latest stored value into later years as it " + "grows lifetime_isa_balance. In a later year the model sums the balances " + "instead where the data hold them for a later year than the total, or where " + "a reform changes what household_lifetime_isa_balance sums or stops " + "lifetime_isa_balance reading its inputs. No means test reads it." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index 41e572e71d..ffa40e9222 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -7,11 +7,12 @@ class stocks_and_shares_isa(Variable): "Amount held in stocks and shares (investment) Individual Savings Accounts: " "stocks_and_shares_isa_balance summed over the household's members. Part of " "corporate_wealth, so the two must never be summed. A dataset that stores " - "this column keeps the stored value, and in later years the model grows the " - "latest stored value as it grows corporate_wealth, so a reform that gives " - "stocks_and_shares_isa_balance a formula leaves a stored total unchanged. " - "Balances that the data hold for a later year than the total replace it with " - "their sum. A household input in a situation applies only to its own year." + "this column keeps the stored value, and the model grows the latest stored " + "value into later years as it grows corporate_wealth. In a later year the " + "model sums the balances instead where the data hold them for a later year " + "than the total, or where a reform changes what stocks_and_shares_isa sums " + "or stops stocks_and_shares_isa_balance reading its inputs. A household " + "input in a situation applies only to its own year." ) entity = Household definition_period = YEAR From 7c2b43d6901a653cb167f6288659636c388238dc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:21:33 -0400 Subject: [PATCH 14/19] Grow the several-parts test total from its last stored year Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_stored_household_totals.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index ebdca3f0f3..d961e898d5 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -803,8 +803,10 @@ def test_a_total_of_several_parts_grows_only_by_an_index_they_share(monkeypatch) ) later = PROJECTION_END_YEAR + 1 assert simulation.calculate("cash_isa", later)[0] == 0 + stored = simulation.calculate("stocks_and_shares_isa", PROJECTION_END_YEAR)[0] + assert stored > 7_000 assert simulation.calculate("stocks_and_shares_isa", later)[0] == pytest.approx( - 7_000 * _growth(simulation, "savings", later, base=DATA_YEAR), rel=1e-6 + stored * _growth(simulation, "savings", later), rel=1e-6 ) From e421eeb2d2378f5363547548b28dcbcd8e8a49bb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:43:27 -0400 Subject: [PATCH 15/19] Say which reforms leave a stored total to Core in its comment Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/data/economic_assumptions.py | 6 +++--- policyengine_uk/utils/measurement_branches.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 5984d1ce19..fd1f7c16fa 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -178,9 +178,9 @@ def growth_rate( # `uprating`. So in a simulation built from data, Simulation grows the latest # earlier stored total by the index its parts declare, as Core grows the # parts, while the total still means what the data stored: a sum of exactly -# these parts, each a plain input. Core sums the parts where the variable -# differs, where the data hold no total for that year or an earlier one, or -# where they hold the parts for a later year than the total. +# these parts, each a plain input. Core sums the parts where a reform changes +# that sum or a part, where the data hold no total for that year or an +# earlier one, or where they hold the parts for a later year than the total. STORED_HOUSEHOLD_TOTALS = MappingProxyType( { "cash_isa": frozenset({"cash_isa_balance"}), diff --git a/policyengine_uk/utils/measurement_branches.py b/policyengine_uk/utils/measurement_branches.py index 9de444cd95..e752895be4 100644 --- a/policyengine_uk/utils/measurement_branches.py +++ b/policyengine_uk/utils/measurement_branches.py @@ -66,8 +66,8 @@ def drop_calculated_values(simulation) -> None: Inputs are variables without a formula, variables the simulation was built from, and variables set since through set_input, as a scenario's simulation modifier does to formula variables. A stored household total - keeps only the years supplied for it (see drop_calculated_totals). Only - variables the simulation holds values for are visited. + keeps only the years supplied for it (see drop_calculated_totals). It + visits only the variables the simulation holds values for. """ from policyengine_uk.data.economic_assumptions import STORED_HOUSEHOLD_TOTALS From 6cf11d4dddc13b6005b84708271a30f4c64595c2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 07:12:39 -0400 Subject: [PATCH 16/19] Set balances only on a named branch in the stored-totals property test On the default branch Core keeps a value it has uprated for a person balance after a balance for an earlier year changes, as on main, so a total that sums the balances reads it. The property now sets totals on either branch and balances on a named branch, and runs two fixed sequences: the grown totals after inputs for 2030 and 2036, and a balance changed after the sum of balances was calculated. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stored_household_totals.py | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index d961e898d5..919b598ef5 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -18,8 +18,9 @@ - with person balances stored instead, the total equals their sum every year; - the two forms give the same totals and means-test capital when they hold the same amounts; -- after any inputs and calculations, a total keeps every value the data or a - caller supplied, and equals what a fresh calculation gives in other years; +- after totals set for any years, and balances set on a named branch, between + calculations, a total keeps every value the data or a caller supplied, and + equals what a fresh calculation gives in other years; - a grown total grows as Core grows an input with its parts' index; - a reform that changes the sum or a part gets Core's sum, and a reform that declares the same variables again changes nothing. @@ -30,7 +31,7 @@ import numpy as np import pandas as pd import pytest -from hypothesis import given, settings +from hypothesis import example, given, settings from hypothesis import strategies as st from policyengine_core.parameters import ParameterNode from policyengine_core.periods import period @@ -926,23 +927,56 @@ def test_stored_totals_and_person_balances_agree_in_every_year(households): INPUT_YEARS = st.integers(min_value=DATA_YEAR + 1, max_value=YEARS[-1]) -STEPS = st.lists( - st.one_of( - st.tuples(st.just("calculate"), INPUT_YEARS), - st.tuples(st.just("cash_isa"), INPUT_YEARS, AMOUNTS), - st.tuples(st.just("cash_isa_balance"), INPUT_YEARS, AMOUNTS), +TOTAL_STEPS = ( + st.tuples(st.just("calculate"), INPUT_YEARS), + st.tuples(st.just("cash_isa"), INPUT_YEARS, AMOUNTS), +) +BALANCE_STEP = st.tuples(st.just("cash_isa_balance"), INPUT_YEARS, AMOUNTS) +# Core keeps a value it has uprated for a person balance on the default +# branch when a balance for an earlier year changes there, so a total that +# sums the balances would read it. Balances change only on a named branch, +# where Core drops what depends on them. +RUNS = st.one_of( + st.tuples( + st.just(False), st.lists(st.one_of(*TOTAL_STEPS), min_size=1, max_size=8) + ), + st.tuples( + st.just(True), + st.lists(st.one_of(*TOTAL_STEPS, BALANCE_STEP), min_size=1, max_size=8), ), - min_size=1, - max_size=8, ) @PROPERTY_SETTINGS -@given(steps=STEPS, on_branch=st.booleans()) -def test_a_total_after_any_inputs_equals_a_fresh_calculation(steps, on_branch): - # Totals or balances set for any year, between calculations of any year: - # each calculated total equals what the same simulation gives with every - # calculated value dropped, and every supplied total keeps its value. +@given(run=RUNS) +@example( + run=( + False, + [ + ("calculate", 2035), + ("cash_isa", 2030, 10_000.0), + ("calculate", 2035), + ("calculate", 2037), + ("cash_isa", 2036, 20_000.0), + ], + ) +) +@example( + run=( + True, + [ + ("cash_isa_balance", 2034, 100.0), + ("calculate", 2040), + ("cash_isa_balance", 2035, 200.0), + ], + ) +) +def test_a_total_after_any_inputs_equals_a_fresh_calculation(run): + # Totals set for any year, and on a named branch balances too, between + # calculations of any year: each calculated total equals what the same + # simulation gives with every calculated value dropped, and every + # supplied total keeps its value. + on_branch, steps = run simulation = Simulation( dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) ) From 2003d18dce9a5eb048d0a83649613c0cfeb889a8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 17:15:38 -0400 Subject: [PATCH 17/19] Leave a stored total to Core once a reform abolishes its part A reform can abolish a part for a period through gov.abolitions, which the stored-total gate didn't read: the total kept growing while its part and Core's sum fell to zero. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/simulation.py | 15 +++++++-- .../tests/test_stored_household_totals.py | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index b653ca832c..5418985537 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -744,14 +744,22 @@ def _latest_source(self, name, period): source = self._isa_latest_input(name, period) return source + def _abolished(self, name, period) -> bool: + # Core's own test for gov.abolitions, which a reform may supply and + # the baseline does not: without the node, nothing is abolished. + try: + return bool(self.tax_benefit_system.parameters(period).gov.abolitions[name]) + except Exception: + return False + def _stands_for_stored_parts(self, variable, period) -> bool: # A stored total means the sum of the parts STORED_HOUSEHOLD_TOTALS # names for it, each a plain input. Grow it only while the variable # still sums exactly those parts, each once, with nothing subtracted, # and each part still reads its inputs: no formula, sum or subtraction - # for this period, no defined_for mask, not neutralized. Anything a - # reform changes there leaves the total to Core, as in data with no - # stored total. + # for this period, no defined_for mask, not neutralized or abolished. + # Anything a reform changes there leaves the total to Core, as in data + # with no stored total. parts = STORED_HOUSEHOLD_TOTALS.get(variable.name) if ( parts is None @@ -770,6 +778,7 @@ def _stands_for_stored_parts(self, variable, period) -> bool: or part.subtracts or part.defined_for is not None or part.is_neutralized + or self._abolished(name, period) ): return False return True diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index 919b598ef5..a6e04c1bdc 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -685,6 +685,39 @@ def test_core_sums_a_total_whose_sum_or_part_a_reform_changes(sum_change, part_c assert total != pytest.approx(grown, rel=1e-3), (name, year) +def _abolish_part_after_the_horizon(system): + system.parameters.gov.add_child( + "abolitions", + ParameterNode( + "gov.abolitions", + data={ + "cash_isa_balance": { + "values": {f"{PROJECTION_END_YEAR + 1}-01-01": True} + } + }, + ), + ) + + +def test_a_total_stops_growing_from_the_year_a_reform_abolishes_its_part(): + # Core reads an abolished part as zero from the year the abolition starts, + # so the stored total stands until then and Core's sum takes over from it. + simulation = Simulation( + reform=_reform(_abolish_part_after_the_horizon), + dataset=_dataset( + [{"cash_isa": 5_000.0, "cash_isa_balance": 400.0}], + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ), + ) + stored = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert stored == simulation.baseline.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert stored > 5_000.0 + for year in (PROJECTION_END_YEAR + 1, YEARS[-1]): + assert simulation.calculate("cash_isa_balance", year).sum() == 0, year + assert simulation.calculate("cash_isa", year)[0] == 0, year + + def test_a_part_formula_takes_over_from_the_year_it_starts(): # Until its formula starts, the part reads its inputs, so the stored # total stands; a part with no index holds it flat, as Core holds the From bbab680d5dec4c896a8a5b2f748bc248b43c9ae7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 18:04:05 -0400 Subject: [PATCH 18/19] Leave a total to Core when its part stops reading annual inputs A reform can give a part another definition period, so the gate read a period Core doesn't, or leave a part with no index and no carry-over, so Core reads nothing for it. Either way the total kept growing while the part and Core's sum gave zero. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/simulation.py | 18 ++++++++--- .../tests/test_stored_household_totals.py | 31 ++++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 5418985537..d7113799b6 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -752,14 +752,22 @@ def _abolished(self, name, period) -> bool: except Exception: return False + def _carries_over(self, variable) -> bool: + # Core's own test for carrying over an input with no index. + return ( + self.tax_benefit_system.auto_carry_over_input_variables + and variable.calculate_output is None + ) + def _stands_for_stored_parts(self, variable, period) -> bool: # A stored total means the sum of the parts STORED_HOUSEHOLD_TOTALS # names for it, each a plain input. Grow it only while the variable # still sums exactly those parts, each once, with nothing subtracted, - # and each part still reads its inputs: no formula, sum or subtraction - # for this period, no defined_for mask, not neutralized or abolished. - # Anything a reform changes there leaves the total to Core, as in data - # with no stored total. + # and each part still reads its inputs: the total's own period, no + # formula, sum or subtraction for this period, no defined_for mask, not + # neutralized or abolished, and an index or Core's carry-over to bring + # an earlier input forward. Anything a reform changes there leaves the + # total to Core, as in data with no stored total. parts = STORED_HOUSEHOLD_TOTALS.get(variable.name) if ( parts is None @@ -773,12 +781,14 @@ def _stands_for_stored_parts(self, variable, period) -> bool: part = self.tax_benefit_system.variables.get(name) if ( part is None + or part.definition_period != variable.definition_period or part.get_formula(period) is not None or part.adds or part.subtracts or part.defined_for is not None or part.is_neutralized or self._abolished(name, period) + or not (part.uprating is not None or self._carries_over(part)) ): return False return True diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index a6e04c1bdc..f3d6cb6274 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -33,11 +33,16 @@ import pytest from hypothesis import example, given, settings from hypothesis import strategies as st +from policyengine_core.model_api import ( + calculate_output_add, + set_input_dispatch_by_period, +) from policyengine_core.parameters import ParameterNode from policyengine_core.periods import period from policyengine_core.reforms import Reform from policyengine_core.simulations import Simulation as CoreSimulation from policyengine_core.simulations.simulation import _uprating_index_value +from policyengine_core.variables import QuantityType from policyengine_uk import Simulation from policyengine_uk.data.dataset_schema import ( @@ -50,7 +55,7 @@ _uprating_index_map, stored_total_uprating, ) -from policyengine_uk.model_api import YEAR, Household, Person, Variable +from policyengine_uk.model_api import MONTH, YEAR, Household, Person, Variable from policyengine_uk.scenarios import no_economic_assumptions pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -619,6 +624,7 @@ class other_isa_holdings(Variable): ), } PART = "stocks_and_shares_isa_balance" +PART_INDEX = "gov.economic_assumptions.indices.obr.per_capita.gdp" PART_CHANGES = { "a_formula": lambda system: system.replace_variable( _redeclared(PART, formula=_a_fixed_balance) @@ -633,6 +639,22 @@ class other_isa_holdings(Variable): _updated(PART, defined_for="is_child") ), "neutralization": lambda system: system.neutralize_variable(PART), + "a_monthly_period": lambda system: system.replace_variable( + _redeclared( + PART, + definition_period=MONTH, + quantity_type=QuantityType.STOCK, + set_input=set_input_dispatch_by_period, + uprating=PART_INDEX, + ) + ), + "no_index_and_a_calculated_output": lambda system: system.replace_variable( + _redeclared(PART, calculate_output=calculate_output_add) + ), + "no_index_and_no_carry_over": lambda system: ( + system.replace_variable(_redeclared(PART)), + setattr(system, "auto_carry_over_input_variables", False), + ), } CHANGE_PAIRS = [ (sum_change, list(PART_CHANGES)[index % len(PART_CHANGES)]) @@ -648,9 +670,10 @@ class other_isa_holdings(Variable): def test_core_sums_a_total_whose_sum_or_part_a_reform_changes(sum_change, part_change): # A stored total stands for the sum of exactly the parts # STORED_HOUSEHOLD_TOTALS names, each a plain input. A reform that sums - # anything else, or that calculates, masks or neutralizes a part, gets - # Core's own calculation in each year the data hold no total, as on data - # that store no total. Years the data hold keep the stored total. + # anything else, or that calculates, masks or neutralizes a part, gives it + # another period or stops Core bringing its inputs forward, gets Core's own + # calculation in each year the data hold no total, as on data that store no + # total. Years the data hold keep the stored total. reform = _reform( _add_parameters, lambda system: system.add_variable(other_isa_holdings), From 9def3670d166dbcfafc9995809bd3eb56db63b7e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 20:04:34 -0400 Subject: [PATCH 19/19] Leave a total to Core when its part casts, waits or holds another unit Core casts each uprated part to its declared type, refuses a part whose prerequisite hasn't run, and carries a balance stored in another unit flat. Growing the total did none of these. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/simulation.py | 16 +++- .../tests/test_stored_household_totals.py | 81 +++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index d7113799b6..7e87c950d0 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -763,8 +763,9 @@ def _stands_for_stored_parts(self, variable, period) -> bool: # A stored total means the sum of the parts STORED_HOUSEHOLD_TOTALS # names for it, each a plain input. Grow it only while the variable # still sums exactly those parts, each once, with nothing subtracted, - # and each part still reads its inputs: the total's own period, no - # formula, sum or subtraction for this period, no defined_for mask, not + # and each part still reads its inputs: the total's own period, a + # float Core doesn't round, no computation prerequisite, no formula, + # sum or subtraction for this period, no defined_for mask, not # neutralized or abolished, and an index or Core's carry-over to bring # an earlier input forward. Anything a reform changes there leaves the # total to Core, as in data with no stored total. @@ -782,6 +783,8 @@ def _stands_for_stored_parts(self, variable, period) -> bool: if ( part is None or part.definition_period != variable.definition_period + or part.value_type is not float + or part.requires_computation_after or part.get_formula(period) is not None or part.adds or part.subtracts @@ -808,9 +811,14 @@ def _stored_household_total(self, variable, period): return None for part in variable.adds: part_source = self._isa_latest_input(part, period) - if part_source is not None and part_source.start > source.start: + if part_source is None: + continue + native = self.tax_benefit_system.variables[part].definition_period + if part_source.start > source.start or part_source.unit != native: # Person balances stored for a later year than the total are - # the newer observation, so their sum stands. + # the newer observation, and Core carries a balance stored in + # another unit flat rather than uprating it, so their sum + # stands. return None try: index_name = stored_total_uprating(self.tax_benefit_system, variable.name) diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py index f3d6cb6274..468ff7c971 100644 --- a/policyengine_uk/tests/test_stored_household_totals.py +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -57,6 +57,7 @@ ) from policyengine_uk.model_api import MONTH, YEAR, Household, Person, Variable from policyengine_uk.scenarios import no_economic_assumptions +from policyengine_uk.utils.scenario import Scenario pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -708,6 +709,86 @@ def test_core_sums_a_total_whose_sum_or_part_a_reform_changes(sum_change, part_c assert total != pytest.approx(grown, rel=1e-3), (name, year) +@pytest.mark.parametrize("value_type", [int, bool], ids=["integer", "boolean"]) +def test_core_sums_a_total_whose_part_a_reform_casts(value_type): + # Core casts each part to its declared type after uprating it, and growing + # the total can't round each part as that cast does. + reform = _reform( + lambda system: system.replace_variable( + _redeclared( + "cash_isa_balance", + value_type=value_type, + default_value=value_type(0), + uprating=PART_INDEX, + ) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 1.0, "cash_isa_balance": 1.0, "share": 0.25}], + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ), + ) + year = PROJECTION_END_YEAR + 1 + total = simulation.calculate("cash_isa", year)[0] + assert total == _core_sum(simulation, "cash_isa", year)[0] + assert total == int(total) + + +def test_a_part_that_needs_an_earlier_calculation_still_raises(): + # Core refuses a variable whose prerequisite hasn't run, and growing the + # total must not hide that refusal. + reform = _reform( + lambda system: system.update_variable( + _updated("cash_isa_balance", requires_computation_after="income_tax") + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 5_000.0, "share": 0.25}], household_columns=["cash_isa"] + ), + ) + with pytest.raises(ValueError, match="income_tax"): + simulation.calculate("cash_isa", PROJECTION_END_YEAR + 1) + + +def _keep_input_unit(holder, requested_period, values): + holder._set(requested_period, values) + + +@pytest.mark.parametrize( + "handler", [None, _keep_input_unit], ids=["no_handler", "custom_handler"] +) +def test_a_balance_stored_in_another_unit_leaves_the_total_to_core(handler): + # Core uprates only earlier inputs in the part's own unit, so it carries a + # monthly balance flat where the total would grow. + class keep_units(Reform): + def apply(self): + self.replace_variable( + _redeclared( + "cash_isa_balance", + quantity_type=QuantityType.STOCK, + uprating=PART_INDEX, + set_input=handler, + ) + ) + + scenario = Scenario.from_reform(keep_units) + scenario.applied_before_data_load = True + simulation = Simulation( + dataset=_one_person_frame(cash_isa__2025=3_000.0), scenario=scenario + ) + simulation.set_input("cash_isa_balance", "2025-01", np.array([3_000.0])) + year = PROJECTION_END_YEAR + 1 + assert simulation.calculate("cash_isa", year)[0] == pytest.approx(3_000) + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + _core_sum(simulation, "cash_isa", year)[0] + ) + + def _abolish_part_after_the_horizon(system): system.parameters.gov.add_child( "abolitions",