diff --git a/changelog.d/2740.fixed.md b/changelog.d/2740.fixed.md new file mode 100644 index 0000000000..cd5fc9bd0a --- /dev/null +++ b/changelog.d/2740.fixed.md @@ -0,0 +1 @@ +Keep the household ISA totals a dataset stores after the years it holds, grown like the person balances they sum, so means-tested capital no longer drops by those holdings from 2033. diff --git a/docs/book/assumptions/isa-inputs-and-capital-flows.md b/docs/book/assumptions/isa-inputs-and-capital-flows.md index b64209bcd7..ce1d7c2c18 100644 --- a/docs/book/assumptions/isa-inputs-and-capital-flows.md +++ b/docs/book/assumptions/isa-inputs-and-capital-flows.md @@ -100,12 +100,16 @@ established by the household tests. `cash_isa_balance`, and `stocks_and_shares_isa` sums their `stocks_and_shares_isa_balance`, as `household_lifetime_isa_balance` sums `lifetime_isa_balance`. A dataset or situation that stores a household total -keeps it: the total is recomputed only when it is not stored, and a stored -total is projected with `savings` or `corporate_wealth`. A household input is -used only for the year it is given for; the person balances carry the -variable-level uprating to later years. The means tests read the household -totals, so a person's balance is pooled with the household's other financial -capital rather than attributed to their own benefit unit. +keeps it: the model sums the person balances only for a year with no stored +total. In each later year the model grows the latest total that a dataset +stores by the index of `savings`, `corporate_wealth` or `lifetime_isa_balance` +respectively, including years after the dataset projection ends, unless the +dataset holds the person balances for a later year than the total, or a reform +changes what the total sums or stops a person balance reading its inputs. A +household input in a situation applies only to its own year; the person +balances carry the variable-level uprating to later years. The means tests read +the household totals, so they pool a person's balance with the household's +other financial capital rather than attribute it to their own benefit unit. Ordinary accessible cash ISA balances enter the capital sources for Universal Credit, Housing Benefit, Pension Credit, Income Support, income-based JSA and diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 3236c3edf6..f9d7a2419d 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -156,10 +156,20 @@ Producers can export per-person `cash_isa_balance` and `stocks_and_shares_isa_balance`. The household `cash_isa` and `stocks_and_shares_isa` sum them, but a stored household column takes precedence, so existing datasets that store only the household totals keep -them, projected with `savings` and `corporate_wealth` through -`uprating_indices.yaml`. A producer that exports the person balances should -either stop exporting the household totals or keep them equal to the sums; -the model does not reconcile the two. +them in every year. `uprating_indices.yaml` projects them with `savings` and +`corporate_wealth` through the years the dataset projection covers. In each +year a data source holds no total for, the simulation grows the latest +earlier stored total by the index its person balances declare +(`STORED_HOUSEHOLD_TOTALS` in `data/economic_assumptions.py`, which also covers +`household_lifetime_isa_balance`, with the person balance each total sums). +Person balances that the data hold for a later year than the total replace it +with their sum, and so does any reform that changes what a total sums or +stops one of its person balances reading its inputs. A situation differs: a +household total given in one applies only to its own year. An input frame is +data, so a situation reloaded from its input frame grows such a total in +later years. A producer that exports the person balances should either stop +exporting the household totals or keep them equal to the sums; the model does +not reconcile the two. `cash_isa_subscriptions` and `stocks_and_shares_isa_subscriptions` are annual person flows, summed over all of a person's accounts of each type, that diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index cd2a64d524..fd1f7c16fa 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -20,6 +20,7 @@ ) import hashlib +from types import MappingProxyType import yaml from policyengine_core.parameters import ParameterNode @@ -165,10 +166,29 @@ def growth_rate( # which the Enhanced FRS does not store, so it falls to -1); one whose # variable has an `uprating` attribute is scaled by that index; any other, # rent and council tax among them, is held at its last projected value. +# Simulation grows the household totals in STORED_HOUSEHOLD_TOTALS itself. # A finite CGT scenario can shorten the whole dataset horizon; it never # extends beyond the evidence declared by that scenario. PROJECTION_END_YEAR = 2032 +# Household totals that sum person inputs (`adds`) but that a dataset can +# store itself (#2740), each with the parts it stands for. For a year with no +# stored total, Core runs the sum, which falls to zero in data that store +# only the total, and Core does not let a variable declare both `adds` and +# `uprating`. So in a simulation built from data, Simulation grows the latest +# earlier stored total by the index its parts declare, as Core grows the +# parts, while the total still means what the data stored: a sum of exactly +# these parts, each a plain input. Core sums the parts where a reform changes +# that sum or a part, where the data hold no total for that year or an +# earlier one, or where they hold the parts for a later year than the total. +STORED_HOUSEHOLD_TOTALS = MappingProxyType( + { + "cash_isa": frozenset({"cash_isa_balance"}), + "stocks_and_shares_isa": frozenset({"stocks_and_shares_isa_balance"}), + "household_lifetime_isa_balance": frozenset({"lifetime_isa_balance"}), + } +) + # Before this horizon was extended, Core uprated unmapped inputs from 2030. # Preserve that behavior in the newly materialized years without changing # projections for earlier years. Correcting historical mapping omissions is @@ -194,6 +214,25 @@ def growth_rate( YEAR_SPECIFIC_INPUTS = frozenset({"unused_pension_annual_allowance"}) +def stored_total_uprating(tax_benefit_system, name: str) -> str | None: + """The index that grows a stored household total: its parts' ``uprating``. + + Growing the total like its parts keeps a stored total equal to the sum it + replaces. ``None`` holds the total flat, as Core holds parts with no + ``uprating``. Parts that declare different indices raise ``ValueError``. + """ + indices = { + tax_benefit_system.variables[part].uprating + for part in tax_benefit_system.variables[name].adds + } + if len(indices) != 1: + raise ValueError( + f"The parts of {name} declare different uprating indices: " + f"{sorted(map(str, indices))}." + ) + return indices.pop() + + def _unmapped_uprating_indices(uprating): from policyengine_uk.system import system diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index 1c63a4598f..32ce0972a6 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -112,7 +112,9 @@ gov.economic_assumptions.yoy_growth.obr.per_capita.gdp: - benunit_reported_capital # cash_isa and stocks_and_shares_isa sum the person balances listed with # them; they are listed so a dataset that stores the household totals -# grows them like their parts. +# grows them like their parts. Past the projected years, Simulation grows +# each stored total in STORED_HOUSEHOLD_TOTALS, household_lifetime_isa_balance +# among them, the same way. - cash_isa - cash_isa_balance - corporate_wealth diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index e41093f979..7e87c950d0 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -38,8 +38,10 @@ from policyengine_uk.utils.scenario import ConstructionReformState, Scenario from policyengine_uk.data.economic_assumptions import ( BEREAVEMENT_SPLIT_INPUTS, + STORED_HOUSEHOLD_TOTALS, extend_single_year_dataset, reset_growthfactor_uprating, + stored_total_uprating, uprating_input_fingerprint, ) from policyengine_uk.data.dataset_sources import materialize_gcs_dataset_url @@ -83,6 +85,7 @@ from policyengine_uk.utils.data_source import built_from_data from policyengine_uk.utils.input_dataframe_export import input_dataframe from policyengine_uk.utils.measurement_branches import ( + drop_calculated_totals, drop_calculated_values, retained_input_variables, ) @@ -706,7 +709,135 @@ def _isa_stored_exemption(self, period): # anchor when translating that stored amount in either direction. return self.calculate(TAX_FREE_INTEREST, period_(period).start.period("year")) + def _record_derived(self, variable_name): + # The value depends on which other periods hold inputs, as an uprated + # one does in Core, so an input set for any period can change it. + history = getattr(self, "_get_store_history", None) + if callable(history): + # Store-history tracking is available in newer Core releases; + # the declared minimum already caches formula results without it. + from policyengine_core.data_storage.store_history import ( + next_sequence_number, + ) + + history().record_derived(variable_name, next_sequence_number()) + + def _index_growth(self, index_name, source, period): + # Core's uprating factor: the ratio of the index at the two starts, + # or 1 where the index has no value or is zero at the source. + if index_name is None: + return 1 + index = self.tax_benefit_system.parameters.get_child(index_name) + source_index = _uprating_index_value(index, source.start) + target_index = _uprating_index_value(index, period.start) + return ( + target_index / source_index + if source_index not in (None, 0) and target_index is not None + else 1 + ) + + def _latest_source(self, name, period): + source = self._isa_latest_input(name, period, earlier=True) + if source is None: + # Core first uprates strictly earlier annual inputs, then carries + # an input starting on the query date if there was none earlier. + source = self._isa_latest_input(name, period) + return source + + def _abolished(self, name, period) -> bool: + # Core's own test for gov.abolitions, which a reform may supply and + # the baseline does not: without the node, nothing is abolished. + try: + return bool(self.tax_benefit_system.parameters(period).gov.abolitions[name]) + except Exception: + return False + + def _carries_over(self, variable) -> bool: + # Core's own test for carrying over an input with no index. + return ( + self.tax_benefit_system.auto_carry_over_input_variables + and variable.calculate_output is None + ) + + def _stands_for_stored_parts(self, variable, period) -> bool: + # A stored total means the sum of the parts STORED_HOUSEHOLD_TOTALS + # names for it, each a plain input. Grow it only while the variable + # still sums exactly those parts, each once, with nothing subtracted, + # and each part still reads its inputs: the total's own period, a + # float Core doesn't round, no computation prerequisite, no formula, + # sum or subtraction for this period, no defined_for mask, not + # neutralized or abolished, and an index or Core's carry-over to bring + # an earlier input forward. Anything a reform changes there leaves the + # total to Core, as in data with no stored total. + parts = STORED_HOUSEHOLD_TOTALS.get(variable.name) + if ( + parts is None + or not built_from_data(self) + or variable.get_formula(period) is not None + or variable.subtracts + or sorted(variable.adds or ()) != sorted(parts) + ): + return False + for name in parts: + part = self.tax_benefit_system.variables.get(name) + if ( + part is None + or part.definition_period != variable.definition_period + or part.value_type is not float + or part.requires_computation_after + or part.get_formula(period) is not None + or part.adds + or part.subtracts + or part.defined_for is not None + or part.is_neutralized + or self._abolished(name, period) + or not (part.uprating is not None or self._carries_over(part)) + ): + return False + return True + + def _stored_household_total(self, variable, period): + # A dataset that stores a household total keeps it, projected like + # its parts, in every year (#2740). Core serves the stored total for + # each year the data hold. For a later year it runs the variable's + # sum of person balances, which such data need not hold, so grow the + # latest stored total instead, as Core grows an input with `uprating`. + # Which years hold the total or its parts decides the result. + if not self._stands_for_stored_parts(variable, period): + return None + self._record_derived(variable.name) + source = self._latest_source(variable.name, period) + if source is None: + return None + for part in variable.adds: + part_source = self._isa_latest_input(part, period) + if part_source is None: + continue + native = self.tax_benefit_system.variables[part].definition_period + if part_source.start > source.start or part_source.unit != native: + # Person balances stored for a later year than the total are + # the newer observation, and Core carries a balance stored in + # another unit flat rather than uprating it, so their sum + # stands. + return None + try: + index_name = stored_total_uprating(self.tax_benefit_system, variable.name) + except ValueError: + # Parts that grow by different indices give the total no index of + # its own, so their sum stands. + return None + # The grown total reads none of the parts, so record them: a part + # input that a branch sets later for a later year then replaces it. + for part in variable.adds: + self._record_derived(part) + stored = self.get_holder(variable.name).get_array(source, self.branch_name) + return stored * self._index_growth(index_name, source, period) + def _run_formula(self, variable, population, period): + if variable.name in STORED_HOUSEHOLD_TOTALS: + stored = self._stored_household_total(variable, period) + if stored is not None: + return stored result = super()._run_formula(variable, population, period) if ( result is not None @@ -720,20 +851,8 @@ def _run_formula(self, variable, population, period): # This is fallback, after Core's explicit input/abolition checks and # policy formulas. Preserve the old gross history even past the last # materialized dataset year; new additive records use Core normally. - history = getattr(self, "_get_store_history", None) - if callable(history): - # Store-history tracking is available in newer Core releases; - # the declared minimum already caches formula results without it. - from policyengine_core.data_storage.store_history import ( - next_sequence_number, - ) - - history().record_derived(variable.name, next_sequence_number()) - source = self._isa_latest_input(variable.name, period, earlier=True) - if source is None: - # Core first uprates strictly earlier annual inputs, then carries - # an input starting on the query date if there was none earlier. - source = self._isa_latest_input(variable.name, period) + self._record_derived(variable.name) + source = self._latest_source(variable.name, period) holder = self.get_holder(variable.name) if source is None: gross = holder.default_array() @@ -744,16 +863,7 @@ def _run_formula(self, variable, population, period): == "additive" ): gross = gross + self._isa_stored_exemption(source) - if variable.uprating is not None: - index = self.tax_benefit_system.parameters.get_child(variable.uprating) - source_index = _uprating_index_value(index, source.start) - target_index = _uprating_index_value(index, period.start) - factor = ( - target_index / source_index - if source_index not in (None, 0) and target_index is not None - else 1 - ) - gross = gross * factor + gross = gross * self._index_growth(variable.uprating, source, period) return gross - self.calculate(TAX_FREE_INTEREST, period) @staticmethod @@ -1009,6 +1119,8 @@ def reset_calculations(self): for variable in self.tax_benefit_system.variables: if variable not in retained: self.delete_arrays(variable) + elif variable in STORED_HOUSEHOLD_TOTALS: + drop_calculated_totals(self, variable) def get_known_variables(self): variables = [] diff --git a/policyengine_uk/tests/test_stored_household_totals.py b/policyengine_uk/tests/test_stored_household_totals.py new file mode 100644 index 0000000000..468ff7c971 --- /dev/null +++ b/policyengine_uk/tests/test_stored_household_totals.py @@ -0,0 +1,1137 @@ +"""A dataset that stores a household ISA total keeps it in every year (#2740). + +cash_isa, stocks_and_shares_isa and household_lifetime_isa_balance sum person +balances, but a dataset can store the household total instead. Up to +PROJECTION_END_YEAR the dataset projection grows a stored total; after it, +Simulation grows the latest stored total by the index of its parts, the index +each variable's documentation projects it with, while the variable still sums +the parts STORED_HOUSEHOLD_TOTALS names and each part reads its inputs. A +dataset that stores only the person balances, or stores them for a later year +than the total, gets their sum, and so does a situation for a year it gives no +total for, and a reform that changes the sum or a part. An input frame is +data, so a total it stores grows too. + +Invariants, for any non-negative amounts: + +- a stored total stays positive in every year from 2025 to 2040 when it starts + positive, and from PROJECTION_END_YEAR grows by its declared index; +- with person balances stored instead, the total equals their sum every year; +- the two forms give the same totals and means-test capital when they hold + the same amounts; +- after totals set for any years, and balances set on a named branch, between + calculations, a total keeps every value the data or a caller supplied, and + equals what a fresh calculation gives in other years; +- a grown total grows as Core grows an input with its parts' index; +- a reform that changes the sum or a part gets Core's sum, and a reform that + declares the same variables again changes nothing. +""" + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import example, given, settings +from hypothesis import strategies as st +from policyengine_core.model_api import ( + calculate_output_add, + set_input_dispatch_by_period, +) +from policyengine_core.parameters import ParameterNode +from policyengine_core.periods import period +from policyengine_core.reforms import Reform +from policyengine_core.simulations import Simulation as CoreSimulation +from policyengine_core.simulations.simulation import _uprating_index_value +from policyengine_core.variables import QuantityType + +from policyengine_uk import Simulation +from policyengine_uk.data.dataset_schema import ( + UKMultiYearDataset, + UKSingleYearDataset, +) +from policyengine_uk.data.economic_assumptions import ( + PROJECTION_END_YEAR, + STORED_HOUSEHOLD_TOTALS, + _uprating_index_map, + stored_total_uprating, +) +from policyengine_uk.model_api import MONTH, YEAR, Household, Person, Variable +from policyengine_uk.scenarios import no_economic_assumptions +from policyengine_uk.utils.scenario import Scenario + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +DATA_YEAR = 2025 +YEARS = range(DATA_YEAR, 2041) +CAPITAL_YEARS = (DATA_YEAR, PROJECTION_END_YEAR, PROJECTION_END_YEAR + 1, YEARS[-1]) +# Each total, the person balance it sums, and the variable whose index its +# documentation says the model grows it with. +TOTALS = { + "cash_isa": ("cash_isa_balance", "savings"), + "stocks_and_shares_isa": ("stocks_and_shares_isa_balance", "corporate_wealth"), + "household_lifetime_isa_balance": ("lifetime_isa_balance", "lifetime_isa_balance"), +} +MEANS_TEST_CAPITAL = ( + "uc_assessable_capital", + "housing_benefit_assessable_capital", + "pension_credit_assessable_capital", + "income_support_assessable_capital", + "jsa_income_assessable_capital", + "esa_income_assessable_capital", + "council_tax_reduction_assessable_capital", +) +CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" +PROPERTY_SETTINGS = settings( + max_examples=5, + deadline=None, + derandomize=True, + database=None, +) + + +def _dataset(households, person_columns=(), household_columns=(), year=DATA_YEAR): + """Two adults per household, with the named columns from ``households``. + + Each household is a dict of amounts. A person column ``name`` takes + ``name`` split between the two adults by the household's ``share``. + """ + people = [] + for index, household in enumerate(households): + share = household.get("share", 1.0) + for adult, age in ((0, 30), (1, 28)): + person = { + "person_id": 2 * index + adult + 1, + "person_benunit_id": index + 1, + "person_household_id": index + 1, + "age": age, + } + for name in person_columns: + first = round(household[name] * share, 2) + person[name] = first if adult == 0 else household[name] - first + people.append(person) + count = len(households) + return UKSingleYearDataset( + person=pd.DataFrame(people), + benunit=pd.DataFrame({"benunit_id": range(1, count + 1)}), + household=pd.DataFrame( + { + "household_id": range(1, count + 1), + "household_weight": [1.0] * count, + "region": ["LONDON"] * count, + "tenure_type": ["RENT_PRIVATELY"] * count, + "council_tax": [1_500.0] * count, + "rent": [12_000.0] * count, + "savings": [1_000.0] * count, + **{ + name: [float(household[name]) for household in households] + for name in household_columns + }, + } + ), + fiscal_year=year, + ) + + +def _values(simulation, name, years=YEARS): + return {year: simulation.calculate(name, year).astype(float) for year in years} + + +def _index_growth(simulation, index_name, year, base): + """Growth from ``base`` to ``year`` by the index ``index_name``.""" + index = simulation.tax_benefit_system.parameters.get_child(index_name) + return _uprating_index_value(index, period(year).start) / _uprating_index_value( + index, period(base).start + ) + + +def _growth(simulation, name, year, base=PROJECTION_END_YEAR): + """Growth from ``base`` to ``year`` by the index ``name`` declares.""" + index_name = simulation.tax_benefit_system.variables[name].uprating + return _index_growth(simulation, index_name, year, base) + + +def _core_sum(simulation, name, year): + """What Core calculates for ``name`` in ``year`` from its definition.""" + variable = simulation.tax_benefit_system.variables[name] + population = simulation.get_variable_population(name) + return CoreSimulation._run_formula(simulation, variable, population, period(year)) + + +def _reform(*changes): + """A reform that applies each change to the tax-benefit system.""" + + class reform(Reform): + def apply(self): + for change in changes: + change(self) + + return reform + + +def _redeclared(name, **attributes): + """``name`` as a reform declares it in full, with ``attributes``.""" + from policyengine_uk.system import system + + baseline = system.variables[name] + return type( + name, + (Variable,), + { + "value_type": baseline.value_type, + "entity": Person if baseline.entity.key == "person" else Household, + "definition_period": baseline.definition_period, + "label": baseline.label, + **attributes, + }, + ) + + +def _updated(name, **attributes): + """``name`` as a reform updates it, keeping every other attribute.""" + return type(name, (Variable,), attributes) + + +def test_stored_household_totals_grow_past_the_projection_horizon(): + simulation = Simulation( + dataset=_dataset( + [ + { + "cash_isa": 3_000.0, + "stocks_and_shares_isa": 2_000.0, + "household_lifetime_isa_balance": 4_000.0, + } + ], + household_columns=TOTALS, + ) + ) + capital = _values(simulation, "uc_assessable_capital") + savings = _values(simulation, "savings") + for name, (_, projected_with) in TOTALS.items(): + totals = _values(simulation, name) + for year in YEARS: + if year <= PROJECTION_END_YEAR: + continue + growth = _growth(simulation, projected_with, year) + assert growth > 1 + assert totals[year][0] == pytest.approx( + totals[PROJECTION_END_YEAR][0] * growth, rel=1e-6 + ) + # Savings, the cash ISA and the stocks-and-shares ISA (through + # share_like_capital) make up the capital, each grown alike. + for year in YEARS: + assert capital[year][0] == pytest.approx(6 * savings[year][0], rel=1e-5) + + +@pytest.mark.parametrize( + "scenario", [None, no_economic_assumptions], ids=["current_law", "no_growth"] +) +def test_stored_totals_project_exactly_like_what_they_are_projected_with(scenario): + # The dataset stores cash_isa equal to savings, stocks_and_shares_isa to + # corporate_wealth, and household_lifetime_isa_balance to the one + # lifetime_isa_balance in the household: Core projects the second of each + # pair, Simulation the first. Without economic growth both stay flat. + household = { + "cash_isa": 1_000.0, + "stocks_and_shares_isa": 2_345.67, + "corporate_wealth": 2_345.67, + "household_lifetime_isa_balance": 4_321.09, + "lifetime_isa_balance": 4_321.09, + } + simulation = Simulation( + scenario=scenario, + dataset=_dataset( + [household], + person_columns=["lifetime_isa_balance"], + household_columns=[ + "cash_isa", + "stocks_and_shares_isa", + "corporate_wealth", + "household_lifetime_isa_balance", + ], + ), + ) + for name, other in ( + ("cash_isa", "savings"), + ("stocks_and_shares_isa", "corporate_wealth"), + ("household_lifetime_isa_balance", "lifetime_isa_balance"), + ): + for year in YEARS: + total = simulation.calculate(name, year) + assert total.sum() == simulation.calculate(other, year).sum(), (name, year) + # So the stocks-and-shares ISA stays within corporate_wealth. + for year in YEARS: + share_like = simulation.calculate("share_like_capital", year) + assert share_like[0] == simulation.calculate("corporate_wealth", year)[0] + + +@pytest.mark.parametrize( + "index", [None, CPI, "parts"], ids=["no_index", "cpi", "parts_index"] +) +def test_a_grown_total_grows_as_core_grows_its_part(index): + # The data store a total and the balance it sums for the same year, so + # the total takes precedence and grows by the balance's index, while Core + # grows the balance itself. A reform can give the balance another index, + # or none, when Core carries the balance flat and the total stays flat. + part = "cash_isa_balance" + reform = None + if index != "parts": + reform = _reform( + lambda system: system.replace_variable( + _redeclared(part, uprating=index, default_value=0) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 1_000.0, part: 1_000.0, "share": 0.3}], + person_columns=[part], + household_columns=["cash_isa"], + ), + ) + if index != "parts": + assert simulation.tax_benefit_system.variables[part].uprating == index + for year in YEARS: + balances = simulation.calculate(part, year) + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ), year + later = simulation.calculate("cash_isa", YEARS[-1])[0] + horizon = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + if index is None: + assert later == horizon + else: + assert later > horizon + + +def test_person_balances_still_sum_without_a_stored_total(): + simulation = Simulation( + dataset=_dataset( + [ + { + "cash_isa_balance": 3_000.0, + "stocks_and_shares_isa_balance": 2_000.0, + "lifetime_isa_balance": 4_000.0, + "share": 0.25, + } + ], + person_columns=[part for part, _ in TOTALS.values()], + ) + ) + for name, (part, _) in TOTALS.items(): + for year in YEARS: + balances = simulation.calculate(part, year) + assert balances[0] > 0 and balances[1] > 0 + assert simulation.calculate(name, year)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ) + + +def test_a_stored_zero_total_takes_precedence_over_same_year_balances(): + # A total of zero is an observation, as in the years the data hold: it + # stays zero beside positive balances stored for the same year, in a + # household on its own and beside one with a positive total. + zero = {"cash_isa": 0.0, "cash_isa_balance": 1_000.0, "share": 0.5} + positive = {"cash_isa": 2_000.0, "cash_isa_balance": 500.0, "share": 0.5} + for households in ([zero], [zero, positive]): + simulation = Simulation( + dataset=_dataset( + households, + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ) + ) + for year in CAPITAL_YEARS: + totals = simulation.calculate("cash_isa", year) + assert totals[0] == 0, year + assert simulation.calculate("cash_isa_balance", year)[:2].sum() > 0 + if len(households) == 2: + savings = simulation.calculate("savings", year)[1] + assert totals[1] == pytest.approx(2 * savings, rel=1e-6), year + + +def test_a_situation_total_is_used_only_for_the_year_it_is_given_for(): + simulation = Simulation( + situation={ + "people": { + "adult": { + "age": {DATA_YEAR: 40}, + "cash_isa_balance": {DATA_YEAR: 3_000}, + } + }, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "cash_isa": {DATA_YEAR: 7_000}, + } + }, + } + ) + assert simulation.calculate("cash_isa", DATA_YEAR)[0] == 7_000 + for year in (DATA_YEAR + 1, PROJECTION_END_YEAR + 1): + balance = simulation.calculate("cash_isa_balance", year)[0] + assert balance > 3_000 + assert simulation.calculate("cash_isa", year)[0] == balance + + +def test_a_total_set_on_a_branch_replaces_the_one_a_later_year_grew_from(): + # Like savings, which Core uprates, a grown total depends on which years + # hold a total, so a total set on a branch for a year in between reaches + # a later year the branch has already calculated. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + branch = simulation.get_branch("edited") + between, later = PROJECTION_END_YEAR + 1, PROJECTION_END_YEAR + 3 + for name in ("cash_isa", "savings"): + assert branch.calculate(name, later)[0] > 0 + branch.set_input(name, between, np.array([10_000.0])) + growth = _growth(simulation, "savings", later, base=between) + for name in ("cash_isa", "savings"): + assert branch.calculate(name, later)[0] == pytest.approx( + 10_000 * growth, rel=1e-6 + ) + assert simulation.calculate("cash_isa", later)[0] == pytest.approx( + 3 * simulation.calculate("savings", later)[0], rel=1e-6 + ) + + +def _one_person_frame(**columns): + return pd.DataFrame( + { + "person_id__2025": [1], + "person_benunit_id__2025": [1], + "person_household_id__2025": [1], + **{name: [amount] for name, amount in columns.items()}, + } + ) + + +@pytest.mark.parametrize("branch", [None, "edited"], ids=["default", "named"]) +def test_a_total_set_for_another_year_replaces_a_grown_total_already_calculated( + branch, +): + # The data supply a total for 2025 only. A simulation that has grown it + # to 2035, then gets a total for 2030, grows that one instead; and one + # that has grown the 2030 total to 2037, then gets a total for 2036, + # grows that. The supplied totals keep their values. + simulation = Simulation(dataset=_one_person_frame(cash_isa__2025=3_000.0)) + target = simulation if branch is None else simulation.get_branch(branch) + assert target.calculate("cash_isa", 2035)[0] == pytest.approx( + 3_000 * _growth(simulation, "savings", 2035, base=2025), rel=1e-6 + ) + target.set_input("cash_isa", 2030, np.array([10_000.0])) + assert target.calculate("cash_isa", 2035)[0] == pytest.approx( + 10_000 * _growth(simulation, "savings", 2035, base=2030), rel=1e-6 + ) + assert target.calculate("cash_isa", 2037)[0] == pytest.approx( + 10_000 * _growth(simulation, "savings", 2037, base=2030), rel=1e-6 + ) + target.set_input("cash_isa", 2036, np.array([20_000.0])) + assert target.calculate("cash_isa", 2037)[0] == pytest.approx( + 20_000 * _growth(simulation, "savings", 2037, base=2036), rel=1e-6 + ) + for year, amount in ((2025, 3_000), (2030, 10_000), (2036, 20_000)): + assert target.calculate("cash_isa", year)[0] == amount + + +@pytest.mark.parametrize("branch", [None, "balances"], ids=["default", "named"]) +def test_balances_set_after_a_grown_total_replace_it(branch): + # A grown total reads none of the person balances, so a balance set for a + # later year than the stored total must still reach a year the + # simulation has already calculated. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + target = simulation if branch is None else simulation.get_branch(branch) + between, later = PROJECTION_END_YEAR + 1, PROJECTION_END_YEAR + 3 + grown = target.calculate("cash_isa", later)[0] + assert grown > 3_000 + target.set_input("cash_isa_balance", between, np.array([600.0, 400.0])) + growth = _growth(simulation, "cash_isa_balance", later, base=between) + assert target.calculate("cash_isa", later)[0] == pytest.approx( + 1_000 * growth, rel=1e-6 + ) + if branch is not None: + assert simulation.calculate("cash_isa", later)[0] == pytest.approx( + grown, rel=1e-6 + ) + + +def test_a_grown_total_records_what_it_depends_on(): + # Inside a calculation, an input set on a branch reaches the values the + # branch holds only through Core's records of what each depends on. A + # grown total depends on the total and on its parts in every other year. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + branch = simulation.get_branch("recorded") + later = PROJECTION_END_YEAR + 3 + branch.calculate("cash_isa", later) + grown = branch.get_holder("cash_isa")._stored_sequence_number( + period(later), "recorded" + ) + history = branch._get_store_history() + for name in ("cash_isa", "cash_isa_balance"): + since = history.earliest_dependency(name, period(later - 1)) + assert since is not None and since <= grown, name + + +def test_resets_drop_grown_totals_and_keep_supplied_ones(): + # A stored total counts as an input by name, but the simulation + # calculates it for the years the data do not hold. Resetting the + # calculations deletes those years and keeps every supplied one. + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + holder = simulation.get_holder("cash_isa") + supplied = {year: holder.get_array(period(year)) for year in YEARS} + assert all(supplied[year] is not None for year in range(DATA_YEAR, 2033)) + later = PROJECTION_END_YEAR + 3 + simulation.calculate("cash_isa", later) + assert holder.get_array(period(later)) is not None + simulation.reset_calculations() + assert holder.get_array(period(later)) is None + for year in range(DATA_YEAR, PROJECTION_END_YEAR + 1): + assert holder.get_array(period(year)) == pytest.approx(supplied[year]) + assert holder.get_array(period(DATA_YEAR))[0] == 3_000 + + +def test_person_balances_stored_after_the_total_are_summed_instead(): + # A total and balances that a dataset stores for the same year: the total + # takes precedence, and grows. Balances stored for later years than the + # total are the newer observation, so their sum stands. + def year(columns, person_columns=(), share=0.25, year=DATA_YEAR): + return _dataset( + [{**columns, "share": share}], + person_columns=person_columns, + household_columns=[name for name in columns if name == "cash_isa"], + year=year, + ) + + simulation = Simulation( + dataset=UKMultiYearDataset( + datasets=[ + year( + {"cash_isa": 5_000.0, "cash_isa_balance": 400.0}, + ["cash_isa_balance"], + ), + year({"cash_isa_balance": 600.0}, ["cash_isa_balance"], year=2026), + year({"cash_isa_balance": 800.0}, ["cash_isa_balance"], year=2027), + ] + ) + ) + assert simulation.calculate("cash_isa", DATA_YEAR)[0] == 5_000 + assert simulation.calculate("cash_isa", 2026)[0] == pytest.approx(600) + assert simulation.calculate("cash_isa", 2027)[0] == pytest.approx(800) + balances = simulation.calculate("cash_isa_balance", 2028) + assert balances.sum() > 800 + assert simulation.calculate("cash_isa", 2028)[0] == pytest.approx( + balances.sum(), rel=1e-6 + ) + + same_year = Simulation( + dataset=UKMultiYearDataset( + datasets=[ + year( + {"cash_isa": 5_000.0, "cash_isa_balance": 400.0}, + ["cash_isa_balance"], + ), + ] + ) + ) + growth = _growth(same_year, "savings", 2026, base=DATA_YEAR) + assert same_year.calculate("cash_isa", 2026)[0] == pytest.approx( + 5_000 * growth, rel=1e-6 + ) + + +def test_a_total_stored_in_an_input_frame_grows_like_savings(): + # An input frame is data: a total it stores grows into later years, as a + # total in a situation does not. + frame = pd.DataFrame( + { + "person_id__2025": [1, 2], + "person_benunit_id__2025": [1, 1], + "person_household_id__2025": [1, 1], + "age__2025": [30, 28], + "savings__2025": [1_000.0, 1_000.0], + "cash_isa__2025": [3_000.0, 3_000.0], + } + ) + simulation = Simulation(dataset=frame) + for year in (DATA_YEAR + 1, PROJECTION_END_YEAR + 1): + savings = simulation.calculate("savings", year)[0] + assert savings > 1_000 + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + 3 * savings, rel=1e-6 + ) + + +def _twice_savings(household, period, parameters): + return 2 * household("savings", period) + + +def _a_fixed_balance(person, period, parameters): + return person.filled_array(123.0) + + +def _add_parameters(system): + system.parameters.add_child( + "stored_totals_test", + ParameterNode( + "stored_totals_test", + data={ + "parts": {"values": {"2000-01-01": ["cash_isa_balance"]}}, + "extra": {"values": {"2000-01-01": 100}}, + }, + ), + ) + + +class other_isa_holdings(Variable): + value_type = float + entity = Person + label = "other ISA holdings" + definition_period = YEAR + + +# Changes to what cash_isa sums, and to stocks_and_shares_isa_balance, the +# part stocks_and_shares_isa sums. Each pair runs in one simulation. +SUM_CHANGES = { + "a_formula": lambda system: system.update_variable( + _updated("cash_isa", formula=_twice_savings) + ), + "a_parameter_list": lambda system: system.update_variable( + _updated("cash_isa", adds="stored_totals_test.parts") + ), + "a_parameter_among_the_parts": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "stored_totals_test.extra"]) + ), + "a_part_with_no_index": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "other_isa_holdings"]) + ), + "a_subtraction": lambda system: system.update_variable( + _updated("cash_isa", subtracts=["other_isa_holdings"]) + ), + "an_extra_part": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "stocks_and_shares_isa_balance"]) + ), + "the_part_twice": lambda system: system.update_variable( + _updated("cash_isa", adds=["cash_isa_balance", "cash_isa_balance"]) + ), + "another_part": lambda system: system.update_variable( + _updated("cash_isa", adds=["stocks_and_shares_isa_balance"]) + ), +} +PART = "stocks_and_shares_isa_balance" +PART_INDEX = "gov.economic_assumptions.indices.obr.per_capita.gdp" +PART_CHANGES = { + "a_formula": lambda system: system.replace_variable( + _redeclared(PART, formula=_a_fixed_balance) + ), + "a_sum": lambda system: system.replace_variable( + _redeclared(PART, adds=["cash_isa_balance"]) + ), + "a_subtraction": lambda system: system.replace_variable( + _redeclared(PART, subtracts=["cash_isa_balance"]) + ), + "a_mask": lambda system: system.update_variable( + _updated(PART, defined_for="is_child") + ), + "neutralization": lambda system: system.neutralize_variable(PART), + "a_monthly_period": lambda system: system.replace_variable( + _redeclared( + PART, + definition_period=MONTH, + quantity_type=QuantityType.STOCK, + set_input=set_input_dispatch_by_period, + uprating=PART_INDEX, + ) + ), + "no_index_and_a_calculated_output": lambda system: system.replace_variable( + _redeclared(PART, calculate_output=calculate_output_add) + ), + "no_index_and_no_carry_over": lambda system: ( + system.replace_variable(_redeclared(PART)), + setattr(system, "auto_carry_over_input_variables", False), + ), +} +CHANGE_PAIRS = [ + (sum_change, list(PART_CHANGES)[index % len(PART_CHANGES)]) + for index, sum_change in enumerate(SUM_CHANGES) +] + + +@pytest.mark.parametrize( + "sum_change, part_change", + CHANGE_PAIRS, + ids=[f"sum_{total}-part_{part}" for total, part in CHANGE_PAIRS], +) +def test_core_sums_a_total_whose_sum_or_part_a_reform_changes(sum_change, part_change): + # A stored total stands for the sum of exactly the parts + # STORED_HOUSEHOLD_TOTALS names, each a plain input. A reform that sums + # anything else, or that calculates, masks or neutralizes a part, gives it + # another period or stops Core bringing its inputs forward, gets Core's own + # calculation in each year the data hold no total, as on data that store no + # total. Years the data hold keep the stored total. + reform = _reform( + _add_parameters, + lambda system: system.add_variable(other_isa_holdings), + SUM_CHANGES[sum_change], + PART_CHANGES[part_change], + ) + household = { + "cash_isa": 5_000.0, + "stocks_and_shares_isa": 7_000.0, + "cash_isa_balance": 400.0, + "stocks_and_shares_isa_balance": 1_000.0, + "share": 0.25, + } + simulation = Simulation( + reform=reform, + dataset=_dataset( + [household], + person_columns=["cash_isa_balance", "stocks_and_shares_isa_balance"], + household_columns=["cash_isa", "stocks_and_shares_isa"], + ), + ) + for name in ("cash_isa", "stocks_and_shares_isa"): + stored = simulation.calculate(name, PROJECTION_END_YEAR)[0] + assert stored > household[name] + assert stored == simulation.baseline.calculate(name, PROJECTION_END_YEAR)[0] + for year in (PROJECTION_END_YEAR + 1, YEARS[-1]): + total = simulation.calculate(name, year)[0] + assert total == pytest.approx( + _core_sum(simulation, name, year)[0], rel=1e-6, abs=1e-6 + ), (name, year) + grown = stored * _growth(simulation, "savings", year) + assert total != pytest.approx(grown, rel=1e-3), (name, year) + + +@pytest.mark.parametrize("value_type", [int, bool], ids=["integer", "boolean"]) +def test_core_sums_a_total_whose_part_a_reform_casts(value_type): + # Core casts each part to its declared type after uprating it, and growing + # the total can't round each part as that cast does. + reform = _reform( + lambda system: system.replace_variable( + _redeclared( + "cash_isa_balance", + value_type=value_type, + default_value=value_type(0), + uprating=PART_INDEX, + ) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 1.0, "cash_isa_balance": 1.0, "share": 0.25}], + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ), + ) + year = PROJECTION_END_YEAR + 1 + total = simulation.calculate("cash_isa", year)[0] + assert total == _core_sum(simulation, "cash_isa", year)[0] + assert total == int(total) + + +def test_a_part_that_needs_an_earlier_calculation_still_raises(): + # Core refuses a variable whose prerequisite hasn't run, and growing the + # total must not hide that refusal. + reform = _reform( + lambda system: system.update_variable( + _updated("cash_isa_balance", requires_computation_after="income_tax") + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 5_000.0, "share": 0.25}], household_columns=["cash_isa"] + ), + ) + with pytest.raises(ValueError, match="income_tax"): + simulation.calculate("cash_isa", PROJECTION_END_YEAR + 1) + + +def _keep_input_unit(holder, requested_period, values): + holder._set(requested_period, values) + + +@pytest.mark.parametrize( + "handler", [None, _keep_input_unit], ids=["no_handler", "custom_handler"] +) +def test_a_balance_stored_in_another_unit_leaves_the_total_to_core(handler): + # Core uprates only earlier inputs in the part's own unit, so it carries a + # monthly balance flat where the total would grow. + class keep_units(Reform): + def apply(self): + self.replace_variable( + _redeclared( + "cash_isa_balance", + quantity_type=QuantityType.STOCK, + uprating=PART_INDEX, + set_input=handler, + ) + ) + + scenario = Scenario.from_reform(keep_units) + scenario.applied_before_data_load = True + simulation = Simulation( + dataset=_one_person_frame(cash_isa__2025=3_000.0), scenario=scenario + ) + simulation.set_input("cash_isa_balance", "2025-01", np.array([3_000.0])) + year = PROJECTION_END_YEAR + 1 + assert simulation.calculate("cash_isa", year)[0] == pytest.approx(3_000) + assert simulation.calculate("cash_isa", year)[0] == pytest.approx( + _core_sum(simulation, "cash_isa", year)[0] + ) + + +def _abolish_part_after_the_horizon(system): + system.parameters.gov.add_child( + "abolitions", + ParameterNode( + "gov.abolitions", + data={ + "cash_isa_balance": { + "values": {f"{PROJECTION_END_YEAR + 1}-01-01": True} + } + }, + ), + ) + + +def test_a_total_stops_growing_from_the_year_a_reform_abolishes_its_part(): + # Core reads an abolished part as zero from the year the abolition starts, + # so the stored total stands until then and Core's sum takes over from it. + simulation = Simulation( + reform=_reform(_abolish_part_after_the_horizon), + dataset=_dataset( + [{"cash_isa": 5_000.0, "cash_isa_balance": 400.0}], + person_columns=["cash_isa_balance"], + household_columns=["cash_isa"], + ), + ) + stored = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert stored == simulation.baseline.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert stored > 5_000.0 + for year in (PROJECTION_END_YEAR + 1, YEARS[-1]): + assert simulation.calculate("cash_isa_balance", year).sum() == 0, year + assert simulation.calculate("cash_isa", year)[0] == 0, year + + +def test_a_part_formula_takes_over_from_the_year_it_starts(): + # Until its formula starts, the part reads its inputs, so the stored + # total stands; a part with no index holds it flat, as Core holds the + # part. From the formula's first year, Core sums the parts. + reform = _reform( + lambda system: system.replace_variable( + _redeclared( + "cash_isa_balance", + default_value=0, + formula_2034=_a_fixed_balance, + ) + ) + ) + simulation = Simulation( + reform=reform, + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]), + ) + horizon = simulation.calculate("cash_isa", PROJECTION_END_YEAR)[0] + assert horizon > 3_000 + assert simulation.calculate("cash_isa", 2033)[0] == horizon + for year in (2034, YEARS[-1]): + assert simulation.calculate("cash_isa", year)[0] == pytest.approx(246) + + +@pytest.mark.parametrize("method", ["replace", "update"]) +def test_reforms_that_declare_a_total_and_its_part_again_change_nothing(method): + from policyengine_uk.system import system as baseline_system + + def declare_again(system): + for total, parts in STORED_HOUSEHOLD_TOTALS.items(): + for name in (total, *parts): + if method == "replace": + system.replace_variable(type(baseline_system.variables[name])) + else: + system.update_variable(_updated(name)) + + households = [ + { + "cash_isa": 3_000.0, + "stocks_and_shares_isa": 2_000.0, + "household_lifetime_isa_balance": 900.0, + "lifetime_isa_balance": 900.0, + "share": 0.5, + }, + { + "cash_isa": 0.0, + "stocks_and_shares_isa": 8_000.0, + "household_lifetime_isa_balance": 0.0, + "lifetime_isa_balance": 0.0, + "share": 1.0, + }, + ] + data = dict( + households=households, + person_columns=["lifetime_isa_balance"], + household_columns=TOTALS, + ) + baseline = Simulation(dataset=_dataset(**data)) + reformed = Simulation(reform=_reform(declare_again), dataset=_dataset(**data)) + for name in TOTALS: + for year in YEARS: + assert ( + reformed.calculate(name, year) == baseline.calculate(name, year) + ).all(), (name, year) + for year in CAPITAL_YEARS: + assert ( + reformed.calculate("uc_assessable_capital", year) + == baseline.calculate("uc_assessable_capital", year) + ).all(), year + + +class other_gdp_holdings(Variable): + value_type = float + entity = Person + label = "other holdings that grow with GDP per capita" + definition_period = YEAR + uprating = "gov.economic_assumptions.indices.obr.per_capita.gdp" + + +def test_a_total_of_several_parts_grows_only_by_an_index_they_share(monkeypatch): + # A total can stand for several parts. It grows by their index when they + # share one; parts that grow by different indices give it none, so Core + # sums them. + import policyengine_uk.simulation as uk_simulation + + parts = { + "cash_isa": ["cash_isa_balance", "other_isa_holdings"], + "stocks_and_shares_isa": [ + "stocks_and_shares_isa_balance", + "other_gdp_holdings", + ], + } + monkeypatch.setattr( + uk_simulation, + "STORED_HOUSEHOLD_TOTALS", + { + **STORED_HOUSEHOLD_TOTALS, + **{name: frozenset(names) for name, names in parts.items()}, + }, + ) + reform = _reform( + lambda system: system.add_variable(other_isa_holdings), + lambda system: system.add_variable(other_gdp_holdings), + *( + lambda system, name=name, names=names: system.update_variable( + _updated(name, adds=names) + ) + for name, names in parts.items() + ), + ) + simulation = Simulation( + reform=reform, + dataset=_dataset( + [{"cash_isa": 5_000.0, "stocks_and_shares_isa": 7_000.0}], + household_columns=parts, + ), + ) + later = PROJECTION_END_YEAR + 1 + assert simulation.calculate("cash_isa", later)[0] == 0 + stored = simulation.calculate("stocks_and_shares_isa", PROJECTION_END_YEAR)[0] + assert stored > 7_000 + assert simulation.calculate("stocks_and_shares_isa", later)[0] == pytest.approx( + stored * _growth(simulation, "savings", later), rel=1e-6 + ) + + +def test_parts_with_different_indices_give_a_total_no_index(): + variables = { + "total": SimpleNamespace(adds=["first", "second"]), + "first": SimpleNamespace(uprating="gov.economic_assumptions.indices.a"), + "second": SimpleNamespace(uprating=None), + } + with pytest.raises(ValueError, match="different uprating indices"): + stored_total_uprating(SimpleNamespace(variables=variables), "total") + + +def test_stored_totals_name_the_parts_their_variables_sum(): + # A stored total grows only while its variable sums exactly these parts, + # so the two must not drift apart. + from policyengine_uk.system import system + + for name, parts in STORED_HOUSEHOLD_TOTALS.items(): + adds = system.variables[name].adds + assert isinstance(adds, list) and len(adds) == len(parts), name + assert set(adds) == parts, name + for part in parts: + assert system.variables[part].is_input_variable(), part + + +def test_stored_totals_grow_by_the_index_they_are_projected_with(): + from policyengine_uk.system import system + + for name, (part, projected_with) in TOTALS.items(): + assert STORED_HOUSEHOLD_TOTALS[name] == {part} + index = system.variables[projected_with].uprating + assert index is not None + assert stored_total_uprating(system, name) == index + + +def test_every_projected_household_sum_of_uprated_inputs_is_a_stored_total(): + # A total listed for dataset projection whose parts Core uprates must keep + # a stored value past PROJECTION_END_YEAR too. + from policyengine_uk.system import system + + listed = {name for names in _uprating_index_map().values() for name in names} + sums_of_uprated_inputs = { + name + for name in listed + if isinstance(system.variables[name].adds, list) + and not system.variables[name].formulas + and all( + system.variables[part].is_input_variable() + and system.variables[part].uprating is not None + for part in system.variables[name].adds + ) + } + assert sums_of_uprated_inputs == set(STORED_HOUSEHOLD_TOTALS) + + +AMOUNTS = st.integers(min_value=0, max_value=200_000_000).map(lambda p: p / 100) +HOUSEHOLDS = st.lists( + st.fixed_dictionaries( + { + "cash_isa": AMOUNTS, + "stocks_and_shares_isa": AMOUNTS, + "household_lifetime_isa_balance": AMOUNTS, + "share": st.sampled_from([0.0, 0.3, 0.5, 1.0]), + } + ), + min_size=1, + max_size=6, +) + + +@PROPERTY_SETTINGS +@given(households=HOUSEHOLDS) +def test_stored_totals_and_person_balances_agree_in_every_year(households): + # The totals-only dataset stores the three household totals; it also + # stores the Lifetime ISA balances, which the means tests read from the + # person. The balances-only dataset stores the same amounts per person. + households = [ + {**household, **{part: household[name] for name, (part, _) in TOTALS.items()}} + for household in households + ] + totals_only = Simulation( + dataset=_dataset( + households, + person_columns=["lifetime_isa_balance"], + household_columns=TOTALS, + ) + ) + balances_only = Simulation( + dataset=_dataset( + households, person_columns=[part for part, _ in TOTALS.values()] + ) + ) + members = balances_only.calculate("person_household_id", DATA_YEAR) + for name, (part, projected_with) in TOTALS.items(): + stored = np.array([household[name] for household in households]) + totals = _values(totals_only, name) + summed = _values(balances_only, name) + for year in YEARS: + assert ((totals[year] > 0) == (stored > 0)).all(), (name, year) + balances = balances_only.calculate(part, year) + sums = np.bincount(members - 1, weights=balances, minlength=len(stored)) + assert summed[year] == pytest.approx(sums, rel=1e-6, abs=1e-6) + assert totals[year] == pytest.approx(summed[year], rel=1e-6, abs=1e-6) + if year > PROJECTION_END_YEAR: + growth = _growth(totals_only, projected_with, year) + assert totals[year] == pytest.approx( + totals[PROJECTION_END_YEAR] * growth, rel=1e-6 + ) + # Capital reads the totals compared above in every year; these years + # cover the data year, both sides of the horizon and the last year. + for name in MEANS_TEST_CAPITAL: + for year in CAPITAL_YEARS: + assert totals_only.calculate(name, year) == pytest.approx( + balances_only.calculate(name, year), rel=1e-5, abs=1e-3 + ), (name, year) + + +INPUT_YEARS = st.integers(min_value=DATA_YEAR + 1, max_value=YEARS[-1]) +TOTAL_STEPS = ( + st.tuples(st.just("calculate"), INPUT_YEARS), + st.tuples(st.just("cash_isa"), INPUT_YEARS, AMOUNTS), +) +BALANCE_STEP = st.tuples(st.just("cash_isa_balance"), INPUT_YEARS, AMOUNTS) +# Core keeps a value it has uprated for a person balance on the default +# branch when a balance for an earlier year changes there, so a total that +# sums the balances would read it. Balances change only on a named branch, +# where Core drops what depends on them. +RUNS = st.one_of( + st.tuples( + st.just(False), st.lists(st.one_of(*TOTAL_STEPS), min_size=1, max_size=8) + ), + st.tuples( + st.just(True), + st.lists(st.one_of(*TOTAL_STEPS, BALANCE_STEP), min_size=1, max_size=8), + ), +) + + +@PROPERTY_SETTINGS +@given(run=RUNS) +@example( + run=( + False, + [ + ("calculate", 2035), + ("cash_isa", 2030, 10_000.0), + ("calculate", 2035), + ("calculate", 2037), + ("cash_isa", 2036, 20_000.0), + ], + ) +) +@example( + run=( + True, + [ + ("cash_isa_balance", 2034, 100.0), + ("calculate", 2040), + ("cash_isa_balance", 2035, 200.0), + ], + ) +) +def test_a_total_after_any_inputs_equals_a_fresh_calculation(run): + # Totals set for any year, and on a named branch balances too, between + # calculations of any year: each calculated total equals what the same + # simulation gives with every calculated value dropped, and every + # supplied total keeps its value. + on_branch, steps = run + simulation = Simulation( + dataset=_dataset([{"cash_isa": 3_000.0}], household_columns=["cash_isa"]) + ) + target = simulation.get_branch("edited") if on_branch else simulation + supplied = { + year: target.calculate("cash_isa", year)[0] for year in range(DATA_YEAR, 2033) + } + for step in [*steps, ("calculate", YEARS[-1])]: + name, year = step[:2] + if name == "calculate": + fresh = target.clone() + fresh.drop_computed_arrays() + calculated = target.calculate("cash_isa", year)[0] + assert calculated == fresh.calculate("cash_isa", year)[0], (step, steps) + elif name == "cash_isa": + target.set_input(name, year, np.array([step[2]])) + supplied[year] = np.float32(step[2]) + else: + target.set_input(name, year, np.array([step[2], 0.0])) + for year, amount in supplied.items(): + assert target.calculate("cash_isa", year)[0] == amount, (year, steps) diff --git a/policyengine_uk/utils/measurement_branches.py b/policyengine_uk/utils/measurement_branches.py index 672eca5eac..e752895be4 100644 --- a/policyengine_uk/utils/measurement_branches.py +++ b/policyengine_uk/utils/measurement_branches.py @@ -42,14 +42,35 @@ def retained_input_variables(simulation) -> set: return retained +def drop_calculated_totals(simulation, name: str) -> None: + """Delete the values of a stored household total that the simulation + calculated, keeping those the data or a caller supplied. + + A total in STORED_HOUSEHOLD_TOTALS that the data store counts as an input + by name, but the simulation calculates it for the years the data do not + hold, from the stored years and the parts. Core's own record of inputs + (``_user_input_keys``, checked by ``Holder._stores_user_input``) decides + which values go, as it does for ``drop_computed_arrays``. + """ + simulation.get_holder(name)._drop_computed() + cache = getattr(simulation, "_fast_cache", None) + if cache: + simulation._fast_cache = { + key: value for key, value in cache.items() if key[0] != name + } + + def drop_calculated_values(simulation) -> None: """Delete the values a simulation has calculated, keeping its inputs. Inputs are variables without a formula, variables the simulation was built from, and variables set since through set_input, as a scenario's - simulation modifier does to formula variables. Only variables the - simulation holds values for are visited. + simulation modifier does to formula variables. A stored household total + keeps only the years supplied for it (see drop_calculated_totals). It + visits only the variables the simulation holds values for. """ + from policyengine_uk.data.economic_assumptions import STORED_HOUSEHOLD_TOTALS + supplied = retained_input_variables(simulation) variables = simulation.tax_benefit_system.variables held = { @@ -59,7 +80,11 @@ def drop_calculated_values(simulation) -> None: } for name in held: variable = variables.get(name) - if variable is None or name in supplied or variable.is_input_variable(): + if variable is None: + continue + if name in supplied or variable.is_input_variable(): + if name in STORED_HOUSEHOLD_TOTALS: + drop_calculated_totals(simulation, name) continue simulation.delete_arrays(name) diff --git a/policyengine_uk/variables/household/wealth/cash_isa.py b/policyengine_uk/variables/household/wealth/cash_isa.py index 968a74544e..123d9eec6a 100644 --- a/policyengine_uk/variables/household/wealth/cash_isa.py +++ b/policyengine_uk/variables/household/wealth/cash_isa.py @@ -4,12 +4,13 @@ class cash_isa(Variable): label = "cash ISA holdings" documentation = ( - "Amount held in cash Individual Savings Accounts: cash_isa_balance " - "summed over the household's members. A dataset that stores this " - "column keeps the stored value, and projects it with savings: it is " - "recomputed only when not stored, so a reform or situation that edits " - "cash_isa_balance leaves a stored total unchanged. A household input " - "is used for the year it is given for only." + "Amount held in cash Individual Savings Accounts: cash_isa_balance summed " + "over the household's members. A dataset that stores this column keeps the " + "stored value, and the model grows the latest stored value into later years " + "as it grows savings. In a later year the model sums the balances instead " + "where the data hold them for a later year than the total, or where a reform " + "changes what cash_isa sums or stops cash_isa_balance reading its inputs. A " + "household input in a situation applies only to its own year." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py index 2ec53080cb..44bfbcd872 100644 --- a/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py +++ b/policyengine_uk/variables/household/wealth/household_lifetime_isa_balance.py @@ -5,11 +5,13 @@ class household_lifetime_isa_balance(Variable): label = "household Lifetime ISA balance" documentation = ( "Lifetime ISA balances summed over the household's members. Like " - "lifetime_isa_balance, it is a component of gross_financial_wealth and " - "must never be added to it. A dataset that stores this column keeps " - "the stored value: it is recomputed only when not stored, so a reform " - "or situation that edits lifetime_isa_balance leaves it unchanged. No " - "means test reads it." + "lifetime_isa_balance, it is a component of gross_financial_wealth and must " + "never be added to it. A dataset that stores this column keeps the stored " + "value, and the model grows the latest stored value into later years as it " + "grows lifetime_isa_balance. In a later year the model sums the balances " + "instead where the data hold them for a later year than the total, or where " + "a reform changes what household_lifetime_isa_balance sums or stops " + "lifetime_isa_balance reading its inputs. No means test reads it." ) entity = Household definition_period = YEAR diff --git a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py index d91949f784..ffa40e9222 100644 --- a/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py +++ b/policyengine_uk/variables/household/wealth/stocks_and_shares_isa.py @@ -4,14 +4,15 @@ class stocks_and_shares_isa(Variable): label = "stocks and shares ISA holdings" documentation = ( - "Amount held in stocks and shares (investment) Individual Savings " - "Accounts: stocks_and_shares_isa_balance summed over the household's " - "members. Part of corporate_wealth, so the two must never be summed. " - "A dataset that stores this column keeps the stored value, and " - "projects it with corporate_wealth: it is recomputed only when not " - "stored, so a reform or situation that edits " - "stocks_and_shares_isa_balance leaves a stored total unchanged. A " - "household input is used for the year it is given for only." + "Amount held in stocks and shares (investment) Individual Savings Accounts: " + "stocks_and_shares_isa_balance summed over the household's members. Part of " + "corporate_wealth, so the two must never be summed. A dataset that stores " + "this column keeps the stored value, and the model grows the latest stored " + "value into later years as it grows corporate_wealth. In a later year the " + "model sums the balances instead where the data hold them for a later year " + "than the total, or where a reform changes what stocks_and_shares_isa sums " + "or stops stocks_and_shares_isa_balance reading its inputs. A household " + "input in a situation applies only to its own year." ) entity = Household definition_period = YEAR