From 4fecf28195de39f742558716301652288d99fca9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 02:04:03 -0400 Subject: [PATCH 1/6] Read calendar-year series' years as calendar years, and write generated-series reforms where they are generated A bare-year reform key on gov.economic_assumptions.yoy_growth (and on the indices, and on the household tree, which is never annualised) named the fiscal year from 6 April, so it reached the next calendar year's value: a 2027 earnings growth rate moved the 2028 index and the 2029 triple lock while showing as 2027's. It now names the calendar year. Changes to parameters processing generates (the triple lock rate, the indices, the lagged series) were applied to the annualised tree after processing, so nothing derived from them followed. They are now written just after the step that generates them. Deprecated labour supply alias paths in reform dicts now reach the parameter they copy. Co-Authored-By: Claude Opus 5.5 --- .../create_economic_assumption_indices.py | 4 +- .../gov/economic_assumptions/yoy_growth.yaml | 6 + policyengine_uk/simulation.py | 14 +- policyengine_uk/tax_benefit_system.py | 165 ++++++++++++++++-- policyengine_uk/utils/dated_reforms.py | 120 +++++++++++-- policyengine_uk/utils/scenario.py | 63 +++++-- 6 files changed, 323 insertions(+), 49 deletions(-) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/create_economic_assumption_indices.py b/policyengine_uk/parameters/gov/economic_assumptions/create_economic_assumption_indices.py index f87f0b7f5e..8eb8e80761 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/create_economic_assumption_indices.py +++ b/policyengine_uk/parameters/gov/economic_assumptions/create_economic_assumption_indices.py @@ -10,9 +10,11 @@ def create_economic_assumption_indices( ) -> ParameterNode: econ_assumptions: ParameterNode = parameters.gov.economic_assumptions yoy_growth: ParameterNode = econ_assumptions.yoy_growth + # Index levels dated 1 January, one per calendar year, so a bare-year + # reform key names the calendar year (policyengine_uk.utils.dated_reforms). indices = ParameterNode( name="gov.economic_assumptions.indices", - data={}, + data={"metadata": {"calendar_year_series": True}}, ) econ_assumptions.add_child( "indices", diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 0a437cc2ad..8b67c2f92e 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -33,6 +33,12 @@ # https://obr.uk/box/the-long-run-difference-between-rpi-and-cpi-inflation/ # https://www.bankofengland.co.uk/monetary-policy/inflation +metadata: + # Each value is the growth over a calendar year, dated 1 January, and + # processing reads it there, so a bare-year reform key names the calendar + # year (policyengine_uk/utils/dated_reforms.py). + calendar_year_series: true + obr: rpi: description: Retail price index year-on-year growth. diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 494463d9ca..da0b59b28a 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -64,7 +64,11 @@ canonicalize_lsr_parameter_path, ) -from .tax_benefit_system import CountryTaxBenefitSystem, DEFAULT_DATASET_ENV_VAR +from .tax_benefit_system import ( + CountryTaxBenefitSystem, + DEFAULT_DATASET_ENV_VAR, + parameter_reads_calendar_years, +) def _canonical_parameter_paths(changes: dict) -> dict: @@ -620,7 +624,10 @@ def apply_parameter_changes(self, changes: dict): read) and the parameters are processed as current law is. """ self.apply_parameter_assignments( - read_parameter_changes(_canonical_parameter_paths(changes)) + read_parameter_changes( + _canonical_parameter_paths(changes), + calendar_years=parameter_reads_calendar_years(), + ) ) def apply_scenario_parameters(self, scenario: Scenario) -> None: @@ -635,7 +642,8 @@ def apply_scenario_parameters(self, scenario: Scenario) -> None: assignments = [] if scenario.parameter_changes: assignments += read_parameter_changes( - _canonical_parameter_paths(scenario.parameter_changes) + _canonical_parameter_paths(scenario.parameter_changes), + calendar_years=parameter_reads_calendar_years(), ) assignments += scenario.dated_assignments(order_start=len(assignments)) self.apply_parameter_assignments(assignments) diff --git a/policyengine_uk/tax_benefit_system.py b/policyengine_uk/tax_benefit_system.py index 14e58f88c7..abc6c5c686 100644 --- a/policyengine_uk/tax_benefit_system.py +++ b/policyengine_uk/tax_benefit_system.py @@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple # PolicyEngine core imports -from policyengine_core.parameters import ParameterNode +from policyengine_core.parameters import Parameter, ParameterNode from policyengine_core.parameters.operations.propagate_parameter_metadata import ( propagate_parameter_metadata, ) @@ -50,9 +50,11 @@ ) from policyengine_uk.parameters.gov.simulation.labour_supply_responses.aliases import ( add_lsr_deprecation_aliases, + canonicalize_lsr_parameter_path, ) from policyengine_uk.utils.dated_reforms import ( BACKDATED_FROM, + CALENDAR_YEAR_SERIES, LegacyOverride, ParameterAssignment, Timeline, @@ -89,16 +91,44 @@ _projected_timelines_cache = None -def _add_generated_parameters(parameters: ParameterNode) -> ParameterNode: - parameters = add_private_pension_uprating_factor(parameters) - parameters = add_lagged_earnings(parameters) - parameters = add_lagged_cpi(parameters) - parameters = add_statutory_uprating_inputs(parameters) - parameters = add_triple_lock(parameters) - parameters = add_september_cpi_uprating(parameters) - parameters = create_economic_assumption_indices(parameters) - parameters = add_threshold_indexation(parameters) - parameters = add_lsr_deprecation_aliases(parameters) +# The steps that generate parameters from others, in order. Each may read +# what the steps before it generate. +GENERATION_STEPS = ( + add_private_pension_uprating_factor, + add_lagged_earnings, + add_lagged_cpi, + add_statutory_uprating_inputs, + add_triple_lock, + add_september_cpi_uprating, + create_economic_assumption_indices, + add_threshold_indexation, + add_lsr_deprecation_aliases, +) + + +def _add_generated_parameters( + parameters: ParameterNode, + generated: Optional[List[ParameterAssignment]] = None, +) -> ParameterNode: + """Run the generation steps, writing changes to generated parameters. + + Each change in ``generated`` is written onto its parameter just after + the step that generates it, so the later steps, and uprating, read it. + """ + pending = list(generated or []) + for step in GENERATION_STEPS: + parameters = step(parameters) + if pending: + defines = tree_defines(parameters) + for assignment in [a for a in pending if defines(a.path)]: + write_assignment(parameters.get_child(assignment.path), assignment) + pending.remove(assignment) + if pending: + raise ValueError( + "Could not find the parameter " + + ", ".join(sorted({assignment.path for assignment in pending})) + + "." + ) return parameters @@ -106,6 +136,7 @@ def process_parameter_tree( parameters: ParameterNode, unreformed: Optional[ParameterNode] = None, annualise: bool = True, + generated: Optional[List[ParameterAssignment]] = None, ) -> Tuple[ParameterNode, Dict[str, Timeline]]: """Project and annualise a statutory parameter tree. @@ -129,8 +160,11 @@ def process_parameter_tree( tree carrying dated reform changes passes ``unreformed``, the same tree without them, which the copy is taken from instead (#2188). ``annualise=False`` stops after the projection and adds no copy. + ``generated`` holds the reform's changes to generated parameters, each + written just after the step that generates it (see + ``_add_generated_parameters``). """ - parameters = _add_generated_parameters(parameters) + parameters = _add_generated_parameters(parameters, generated) # Create baseline parameters for reform comparisons if annualise: @@ -370,12 +404,22 @@ def _rebuild(self, raw: ParameterNode, statutory: bool) -> None: if not assignments: self.parameters, self._projected_timelines = process_parameter_tree(raw) return + # Changes to parameters processing generates are written as each is + # generated; the rest go on the statutory timeline first. + defines = tree_defines(raw) + on_timeline = [a for a in assignments if defines(a.path)] + generated = [a for a in assignments if not defines(a.path)] + generates = generated_parameter_defines() + for assignment in generated: + if not generates(assignment.path): + # Not generated either: the error the lookup gives. + raw.get_child(assignment.path) unreformed = raw.clone() - projected = prior_law_projection(assignments, unreformed, statutory) - for assignment in assignments: + projected = prior_law_projection(on_timeline, unreformed, statutory, generated) + for assignment in on_timeline: write_assignment(raw.get_child(assignment.path), assignment, projected) self.parameters, self._projected_timelines = process_parameter_tree( - raw, unreformed + raw, unreformed, generated=generated ) def _apply_legacy_overrides(self) -> None: @@ -476,6 +520,83 @@ def statutory_parameter_defines() -> Callable[[str], bool]: return tree_defines(_statutory_parameters_cache) +# The paths of the parameters the generation steps add to the statutory tree, +# other than the deprecated labour supply aliases. +_generated_paths_cache: Optional[frozenset] = None + + +def _leaf_paths(node: ParameterNode, prefix: str = "") -> List[str]: + paths = [] + for key, child in node.children.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, Parameter): + paths.append(path) + elif isinstance(child, ParameterNode): + paths += _leaf_paths(child, path) + else: + for index, bracket in enumerate(getattr(child, "brackets", [])): + paths += _leaf_paths(bracket, f"{path}[{index}]") + return paths + + +def generated_parameter_defines() -> Callable[[str], bool]: + """Whether processing generates a parameter at a path. + + The deprecated labour supply aliases are left out: a change to one is a + change to the parameter it copies (``canonicalize_lsr_parameter_path``). + """ + global _generated_paths_cache + if _generated_paths_cache is None: + _ensure_caches() + statutory = tree_defines(_statutory_parameters_cache) + generated = _add_generated_parameters(statutory_parameters()) + _generated_paths_cache = frozenset( + path + for path in _leaf_paths(generated) + if not statutory(path) and canonicalize_lsr_parameter_path(path) == path + ) + return _generated_paths_cache.__contains__ + + +def reform_parameter_defines() -> Callable[[str], bool]: + """Whether a reform's change to a path is written on the timeline. + + True for the parameters in the statutory tree and those processing + generates; changes to any other path (one a structural reform adds) are + legacy overrides of the annualised parameters. + """ + statutory = statutory_parameter_defines() + generated = generated_parameter_defines() + return lambda path: statutory(path) or generated(path) + + +def parameter_reads_calendar_years() -> Callable[[str], bool]: + """Whether a parameter's values are for calendar years. + + So a bare-year reform key names the calendar year, not the fiscal year + (see ``policyengine_uk.utils.dated_reforms``). True for a parameter + outside ``gov``, which is never annualised and is read at 1 January, and + for one with ``calendar_year_series`` metadata on it or an ancestor node. + """ + _ensure_caches() + tree = _processed_parameters_cache + + def reads(path: str) -> bool: + try: + node = tree.get_child(path) + except Exception: + return False + if path.split(".")[0] != "gov": + return True + while node is not None: + if (getattr(node, "metadata", None) or {}).get(CALENDAR_YEAR_SERIES): + return True + node = getattr(node, "parent", None) + return False + + return reads + + def projected_timelines() -> Dict[str, Timeline]: """Current law's projected timeline for each gov parameter, by name. @@ -497,6 +618,7 @@ def prior_law_projection( assignments: List[ParameterAssignment], unreformed: ParameterNode, statutory: bool = True, + generated: Optional[List[ParameterAssignment]] = None, ) -> Callable[[str], Optional[Timeline]]: """Prior law's projected timeline, by parameter name, for a reform. @@ -510,7 +632,8 @@ def prior_law_projection( that still change after that depend on each other in a cycle, which is an error. ``unreformed`` is the tree the changes are written onto; ``statutory`` says it is the YAML as loaded, whose projection without - changes is cached. + changes is cached. ``generated`` holds the reform's changes to generated + parameters, which every pass makes too. """ needing = sorted( { @@ -523,8 +646,10 @@ def prior_law_projection( return lambda name: None start = ( projected_timelines() - if statutory - else process_parameter_tree(unreformed.clone(), annualise=False)[1] + if statutory and not generated + else process_parameter_tree( + unreformed.clone(), annualise=False, generated=generated + )[1] ) environment = {path: start.get(path) for path in needing} @@ -541,7 +666,9 @@ def lookup(name: str) -> Optional[Timeline]: write_assignment( tree.get_child(assignment.path), assignment, lookup ) - updated[path] = process_parameter_tree(tree, annualise=False)[1].get(path) + updated[path] = process_parameter_tree( + tree, annualise=False, generated=generated + )[1].get(path) if _same_timelines(updated, environment): return lookup environment = updated diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index 16b6cfab6b..85761e17c9 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -49,7 +49,7 @@ Key reform dict ``parameter_changes`` ``Reform.from_dict`` ===================== ======================== ========================= ========================== ``"start.stop"`` ``[start, stop]`` ``[start, stop]`` ``[start, stop]`` -``"2027"`` or 2027 fiscal year 2027 only fiscal year 2027 only from 1 January 2027 on +``"2027"`` or 2027 fiscal year 2027 only* fiscal year 2027 only* from 1 January 2027 on ``"2027-04-06"`` one day of the that one day from that day on annualised parameter ``"year:2027:2"`` calendar years 2027-28 calendar years 2027-28 calendar years 2027-28 @@ -57,18 +57,57 @@ scalar from 1 January 2023 on 2000 to 2099 2000 to 2099 ===================== ======================== ========================= ========================== +Starred: calendar year 2027 on a calendar-year series (below), where a +``"2027-01-01"`` key also names calendar year 2027. + "From ... on" holds the value flat to 31 December 2100, as editing the annualised parameters did; uprating and indexation do not apply to it. +Calendar-year series +-------------------- + +Some parameters hold one value per calendar year, dated 1 January, and are +read there: the parameters outside ``gov``, which are never annualised and +which formulas read at 1 January, and the parameters carrying +``calendar_year_series`` metadata, on themselves or an ancestor node (the +growth rates in ``gov.economic_assumptions.yoy_growth`` and the indices +processing builds from them). Processing reads the growth rates at +1 January: the indices, the forecasts of the statutory uprating inputs, the +lagged series and the dataset uprating all do. On these parameters the +reform dict and ``parameter_changes`` read a bare year, and a key naming +its 1 January (``"2027-01-01"``), as that calendar year, 1 January to +31 December. The fiscal year from 6 April would reach the next year's value +instead: a 2027 earnings growth rate keyed ``"2027"`` showed as 2027's but +moved the 2028 index, the May to July 2028 earnings input and the April +2029 triple lock. + +Generated series +---------------- + +Processing generates some parameters from others: the triple lock rate +(``gov.economic_assumptions.yoy_growth.triple_lock``), the lagged and +private pension series, the September CPI uprating rate and every index +under ``gov.economic_assumptions.indices``. A change to one is written onto +its timeline just after the step that generates it, so the steps after it, +and uprating, use it: a triple lock rate set for April 2028 uprates the +State Pension, and a changed index uprates the parameters indexed by it. +Each route reads its keys as for the statutory parameters. The deprecated +labour supply aliases (``gov.simulation.labor_supply_responses.*``) are +copies of ``gov.simulation.labour_supply_responses.*``; a change to one is +made to the parameter it copies. + +Legacy overrides +---------------- + A reform-dict key naming a single day or month keeps its legacy meaning: it overrides the annualised parameter on that day or month only, after processing. Formulas read an annual parameter at 1 January, so a ``"2027-01-01"`` key sets fiscal year 2027 and a ``"2027-04-06"`` key changes nothing; ``ReformPeriodWarning`` says so. Use ``"2027-04-06.2100-12-31"`` -for a change from 6 April 2027. A reform-dict path the statutory tree does -not define (one added by a structural reform, or a generated series such as -``gov.economic_assumptions.indices.*``) is also applied to the annualised -parameters, after the structural reforms that come before it. Changes to the +for a change from 6 April 2027. A reform-dict path that is neither in the +statutory tree nor generated by processing (one added by a structural +reform) is also applied to the annualised parameters, after the structural +reforms that come before it. Changes to the annualised parameters always follow the statutory-timeline changes, so a reform-dict parameter that mixes single-day keys with other keys gets the single-day keys last, whatever their order (``ReformPeriodWarning`` says @@ -113,6 +152,9 @@ # Where process_parameter_tree backdates each parameter's first value to. BACKDATED_FROM = "2015-01-01" +# Metadata marking a node or parameter whose values are for calendar years. +CALENDAR_YEAR_SERIES = "calendar_year_series" + REFORM_DICT = "reform_dict" PARAMETER_CHANGES = "parameter_changes" REFORM_FROM_DICT = "reform_from_dict" @@ -159,6 +201,43 @@ def uk_fiscal_year_bounds(year: int) -> Tuple[Instant, Instant]: return instant(f"{year}-04-06"), instant(f"{year + 1}-04-05") +def calendar_year_bounds(year: int) -> Tuple[Instant, Instant]: + """1 January to 31 December of ``year``.""" + return instant(f"{year}-01-01"), instant(f"{year}-12-31") + + +def new_year_day(key: str) -> Optional[int]: + """The year of a key naming 1 January of one year (``"2027-01-01"``).""" + try: + parsed: Period = period(key) + except Exception: + return None + if parsed.unit != "day" or parsed.size != 1: + return None + start = parsed.start + return start.year if (start.month, start.day) == (1, 1) else None + + +def named_year( + key: Any, path: str, calendar_years: Optional[Callable[[str], bool]] +) -> Optional[Tuple[Instant, Instant]]: + """The year a key names, as the bounds a change keyed by it covers. + + A bare year names the UK fiscal year from 6 April, except on a + parameter whose values are for calendar years (``calendar_years(path)``): + there it, or a key naming its 1 January, names that calendar year. + """ + calendar = calendar_years is not None and calendar_years(path) + year = bare_year(key) + if year is not None: + return calendar_year_bounds(year) if calendar else uk_fiscal_year_bounds(year) + if calendar and isinstance(key, str): + year = new_year_day(key) + if year is not None: + return calendar_year_bounds(year) + return None + + def bare_year(key: Any) -> Optional[int]: """The year a bare-year key names, or None.""" if isinstance(key, bool): @@ -202,13 +281,19 @@ def read_reform_dict( defines: Callable[[str], bool], order_start: int = 0, warn: bool = False, + calendar_years: Optional[Callable[[str], bool]] = None, ) -> Tuple[List[ParameterAssignment], List[LegacyOverride]]: """Read a ``Simulation(reform=dict)`` / ``Scenario.from_reform`` dict. - ``defines(path)`` says whether the statutory parameter tree defines a - path. Changes to paths it does not define, and single-day or single-month - keys, are returned as legacy overrides of the annualised parameters; - ``warn`` warns about single-day keys that change no annual value. + ``defines(path)`` says whether a path is written on the statutory + timeline: the statutory parameter tree defines it, or processing + generates it (then the change is written just after it is generated). + Changes to other paths, and single-day or single-month keys, are + returned as legacy overrides of the annualised parameters; ``warn`` + warns about single-day keys that change no annual value. + ``calendar_years(path)`` says whether a parameter's values are for + calendar years, so that a bare year, or a key naming its 1 January, + names that calendar year (see ``named_year``). """ assignments: List[ParameterAssignment] = [] legacy: List[LegacyOverride] = [] @@ -229,11 +314,11 @@ def read_reform_dict( continue for key, value in spec.items(): text = str(key) - year = bare_year(key) + year = named_year(key, path, calendar_years if statutory else None) if "." in text: start, stop = interval_bounds(text) elif year is not None: - start, stop = uk_fiscal_year_bounds(year) + start, stop = year elif is_sub_year(text): if warn: _warn_sub_year_key(path, text) @@ -278,9 +363,14 @@ def _warn_sub_year_key(path: str, key: str) -> None: def read_parameter_changes( - changes: Mapping[str, Any], order_start: int = 0 + changes: Mapping[str, Any], + order_start: int = 0, + calendar_years: Optional[Callable[[str], bool]] = None, ) -> List[ParameterAssignment]: - """Read ``Scenario(parameter_changes=...)``.""" + """Read ``Scenario(parameter_changes=...)``. + + ``calendar_years`` is as for ``read_reform_dict``. + """ assignments: List[ParameterAssignment] = [] order = order_start for path, spec in changes.items(): @@ -292,11 +382,11 @@ def read_parameter_changes( continue for key, value in spec.items(): text = str(key) - year = bare_year(key) + year = named_year(key, path, calendar_years) if "." in text: start, stop = interval_bounds(text) elif year is not None: - start, stop = uk_fiscal_year_bounds(year) + start, stop = year else: start, stop = period_bounds(text) assignments.append(ParameterAssignment(path, start, stop, value, order)) diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index d51ab2d71b..6906021304 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -33,12 +33,39 @@ class DatedReform(BaseModel): reading: Literal["reform_dict", "reform_from_dict"] changes: Dict[str, Any] - def read(self, defines, order_start: int = 0, warn: bool = False): + def read( + self, + defines, + order_start: int = 0, + warn: bool = False, + calendar_years=None, + ): if self.reading == REFORM_DICT: - return read_reform_dict(self.changes, defines, order_start, warn=warn) + return read_reform_dict( + self.changes, + defines, + order_start, + warn=warn, + calendar_years=calendar_years, + ) return read_reform_from_dict(self.changes, defines, order_start) +def _canonical_paths(changes: Dict[str, Any]) -> Dict[str, Any]: + """Changes keyed by canonical parameter paths. + + A deprecated labour supply alias path names the parameter it copies, + which is the one formulas read. + """ + from policyengine_uk.parameters.gov.simulation.labour_supply_responses.aliases import ( + canonicalize_lsr_parameter_path, + ) + + return { + canonicalize_lsr_parameter_path(path): value for path, value in changes.items() + } + + def _is_reform_from_dict(reform: Type[Reform]) -> bool: """Whether a Reform class was built by ``Reform.from_dict``.""" return reform.__qualname__ == "Reform.from_dict..reform" and isinstance( @@ -56,15 +83,20 @@ def _legacy_override_modifier(dated: DatedReform) -> Callable[[Simulation], None """ def modifier(simulation: Simulation) -> None: - from policyengine_uk.tax_benefit_system import statutory_parameter_defines + from policyengine_uk.tax_benefit_system import ( + parameter_reads_calendar_years, + reform_parameter_defines, + ) # Scenario.apply edits an already-built simulation's annualised # parameters, so there every change of the reform is an override. every_change = getattr(simulation, APPLYING_TO_BUILT_SIMULATION, False) - defines = ( - (lambda path: False) if every_change else statutory_parameter_defines() + defines = (lambda path: False) if every_change else reform_parameter_defines() + _, legacy = dated.read( + defines, + warn=not every_change, + calendar_years=parameter_reads_calendar_years(), ) - _, legacy = dated.read(defines, warn=not every_change) if not legacy: return _add_overrides(simulation, legacy) @@ -190,12 +222,20 @@ def combined_modifier(simulation: Simulation) -> None: def dated_assignments(self, order_start: int = 0) -> List[ParameterAssignment]: """The statutory-timeline changes of ``dated_reforms``, in order.""" - from policyengine_uk.tax_benefit_system import statutory_parameter_defines + from policyengine_uk.tax_benefit_system import ( + parameter_reads_calendar_years, + reform_parameter_defines, + ) - defines = statutory_parameter_defines() + defines = reform_parameter_defines() + calendar_years = parameter_reads_calendar_years() assignments: List[ParameterAssignment] = [] for dated in self.dated_reforms or []: - read, _ = dated.read(defines, order_start + len(assignments)) + read, _ = dated.read( + defines, + order_start + len(assignments), + calendar_years=calendar_years, + ) assignments += read return assignments @@ -243,7 +283,8 @@ def from_reform(cls, reform: Union[tuple, dict, Type[Reform]]) -> "Scenario": if isinstance(reform, type) and issubclass(reform, Reform): if _is_reform_from_dict(reform): dated = DatedReform( - reading=REFORM_FROM_DICT, changes=reform.parameter_values + reading=REFORM_FROM_DICT, + changes=_canonical_paths(reform.parameter_values), ) return cls( dated_reforms=[dated], @@ -262,7 +303,7 @@ def modifier(simulation: Simulation) -> None: elif isinstance(reform, dict): for parameter in reform: check_parameter_not_removed(parameter) - dated = DatedReform(reading=REFORM_DICT, changes=dict(reform)) + dated = DatedReform(reading=REFORM_DICT, changes=_canonical_paths(reform)) return cls( dated_reforms=[dated], simulation_modifier=_legacy_override_modifier(dated), From 34d8acc2340a9c058c9ed47cef239fc3be530560 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 02:21:41 -0400 Subject: [PATCH 2/6] Test process-time and generated-series reforms; update the reform docs Example, property and differential tests: the minimum rate and earnings growth move the triple lock by its rule and the State Pension by that rate on every route; every key for a year of growth gives the same parameters and the shown growth is the growth used; a triple lock rate set by reform equals the statutory outturn override; ordinary parameters read every key as before. Co-Authored-By: Claude Opus 5.5 --- .../reform-dict-process-time-params.fixed.md | 1 + docs/book/programs/gov/dwp/state-pension.md | 24 +- docs/book/usage/scenarios.md | 6 +- .../tests/test_process_time_reforms.py | 505 ++++++++++++++++++ 4 files changed, 524 insertions(+), 12 deletions(-) create mode 100644 changelog.d/reform-dict-process-time-params.fixed.md create mode 100644 policyengine_uk/tests/test_process_time_reforms.py diff --git a/changelog.d/reform-dict-process-time-params.fixed.md b/changelog.d/reform-dict-process-time-params.fixed.md new file mode 100644 index 0000000000..34985128fc --- /dev/null +++ b/changelog.d/reform-dict-process-time-params.fixed.md @@ -0,0 +1 @@ +A bare-year reform key on the economic growth rates, the indices built from them and the household parameters now names that calendar year. It used to name the fiscal year from 6 April, which moved the next year: a 2027 earnings growth rate showed as 2027's but moved the 2028 index and the April 2029 triple lock. Reforms to series built during processing (the triple lock rate, the indices, the lagged series) now uprate what is built from them, and deprecated `labor_supply_responses` paths in reform dicts reach the parameters formulas read. diff --git a/docs/book/programs/gov/dwp/state-pension.md b/docs/book/programs/gov/dwp/state-pension.md index 985576b98a..e4c03f4039 100644 --- a/docs/book/programs/gov/dwp/state-pension.md +++ b/docs/book/programs/gov/dwp/state-pension.md @@ -225,12 +225,12 @@ hand, run it with `--gaps-only`. ### Scenarios -A macro scenario applied before the data load that edits calendar-year -growth moves the forecast inputs one for one, and so the triple lock. -Calendar-year series are keyed to 1 January, so key the change -`year:YYYY-01-01:1`: a bare year names the fiscal year from 6 April, which -lands in the next calendar-year value and so moves the following year's -inputs. +A macro scenario that edits calendar-year growth moves the forecast inputs +one for one, and so the triple lock. Calendar-year series are keyed to +1 January, and a bare year (or `year:YYYY-01-01:1`) names that calendar +year: 5% earnings growth in 2027 sets the May to July 2027 input and so the +April 2028 rise. Apply it before the data load to project the data's +incomes with it too. ```python from policyengine_uk.model_api import Scenario @@ -238,7 +238,7 @@ from policyengine_uk.model_api import Scenario scenario = Scenario( parameter_changes={ "gov.economic_assumptions.yoy_growth.obr.average_earnings": { - "year:2027-01-01:1": 0.05, + "2027": 0.05, }, }, applied_before_data_load=True, @@ -265,10 +265,12 @@ Scenario( ) ``` -These changes, and the reform levers below, must go through -`Scenario(parameter_changes=...)`. The rates are built when parameters are -processed, and a `reform=` dictionary edits parameters after that, so it -changes the parameter but not the uprating. +These changes, and the reform levers below, work through +`Scenario(parameter_changes=...)` or a `reform=` dictionary alike: both are +written onto the parameters before the rates are built. A rate can also be +set directly, as `gov.economic_assumptions.yoy_growth.triple_lock` or the +statutory override `triple_lock/outturn.yaml`; either uprates the pension +from that April. ### Reform levers diff --git a/docs/book/usage/scenarios.md b/docs/book/usage/scenarios.md index d50098a3f0..74099afbdb 100644 --- a/docs/book/usage/scenarios.md +++ b/docs/book/usage/scenarios.md @@ -390,12 +390,16 @@ Each way of passing a reform reads period keys as it always has: | Key | `reform=` dict | `parameter_changes` | `Reform.from_dict` | |---|---|---|---| | `"2027-04-06.2028-04-05"` | those dates | those dates | those dates | -| `"2027"` or `2027` | fiscal year 2027 only | fiscal year 2027 only | from 1 January 2027 onwards | +| `"2027"` or `2027` | fiscal year 2027 only (calendar year 2027 on a calendar-year series) | fiscal year 2027 only (calendar year 2027 on a calendar-year series) | from 1 January 2027 onwards | | `"2027-04-06"` | that one day of the annualised value | that one day | from that day onwards | | a value with no period | from 1 January 2023 onwards | 2000 to 2099 | 2000 to 2099 | A single-date key in a `reform=` dict overrides the annualised value on that day only. Formulas read annual values on 1 January, so `"2027-01-01"` sets fiscal year 2027 and `"2027-04-06"` changes nothing. The model logs a warning when a key can change nothing. Use `"start.stop"` keys for dated measures. +Some parameters hold one value per calendar year, dated 1 January, and are read there: the growth rates under `gov.economic_assumptions.yoy_growth`, the indices built from them under `gov.economic_assumptions.indices`, and the `household` parameters, which are not annualised. On these a bare year, or its 1 January, names the calendar year. So `{"gov.economic_assumptions.yoy_growth.obr.average_earnings": {"2027": 0.05}}` sets earnings growth in 2027: the 2027 earnings index, the May to July 2027 earnings input and the April 2028 triple lock all follow it. + +Processing also builds some parameters from others: the triple lock rate (`gov.economic_assumptions.yoy_growth.triple_lock`), the lagged and private pension series, the September CPI uprating rate and the indices. A change to one is written as soon as it is built, so everything built from it follows. `{"gov.economic_assumptions.yoy_growth.triple_lock": {"2028": 0.05}}` uprates the State Pension by 5% in April 2028, exactly as `gov.dwp.state_pension.triple_lock.outturn` set for that April does. A change to an index uprates the parameters indexed by it. The deprecated `gov.simulation.labor_supply_responses` paths change the `labour_supply_responses` parameters they copy. + Changes applied after the data is loaded (the default) leave the data as built, so a changed economic assumption does not re-uprate incomes. Dated changes (`parameter_changes` and `"start.stop"` or bare-year keys) still come before the steps that read parameters while the simulation is built: the Universal Credit rebalancing awards and the structural reforms that parameters switch on. Single-day keys and a scenario's own simulation modifier run after those steps, as they always have. Set `applied_before_data_load=True` on a scenario to project the data with changed economic assumptions too. All of this applies when the scenario or reform is passed to `Simulation(...)`. `Scenario.apply(simulation)` on a simulation that is already built edits its annualised values instead, as it always has: a `"2027-04-06.2028-04-05"` key there overrides those dates of the fiscal-year values, so it does not set fiscal year 2027, and the model logs a warning saying so. Pass the scenario when constructing the simulation for dated measures. diff --git a/policyengine_uk/tests/test_process_time_reforms.py b/policyengine_uk/tests/test_process_time_reforms.py new file mode 100644 index 0000000000..ab6d24107d --- /dev/null +++ b/policyengine_uk/tests/test_process_time_reforms.py @@ -0,0 +1,505 @@ +"""Reforms to parameters that processing reads, or generates, take effect. + +Processing (``process_parameter_tree``) builds the triple lock rate from its +inputs, the economic indices from the growth rates and the uprated amounts +from the indices. A reform to one of those inputs, or to a generated series, +must move what is built from it; an ordinary reform must be unaffected. + +Invariants: + +1. Inputs reach what is built from them: a change to the triple lock's + minimum rate or to calendar-year earnings growth moves the triple lock + rate by its rule and the State Pension amounts by that rate, through the + reform dict and both Scenario timings alike. +2. A year is one year on a calendar-year series: on the growth rates, the + indices and the household tree, a bare year, its 1 January, + ``year:Y-01-01:1`` and ``Y-01-01.Y-12-31`` give identical parameters, and + the value shown for year Y is the value processing used for year Y (the + index grows by it in Y; the May to July Y earnings input moves one for + one; the April Y+1 triple lock follows). +3. Generated series take effect: a triple lock rate set by reform equals + the same rate set through its statutory override (``outturn``), on every + route; a changed index uprates the parameters indexed by it. +4. Ordinary reforms are unaffected (differential): for every parameter that + is not a calendar-year series, the reform dict and ``parameter_changes`` + read every key exactly as before. +5. Monotonicity: the April 2028 State Pension is non-decreasing in the + minimum rate. +6. Baselines stay current law: ``sim.baseline`` and ``parameters.baseline``. +""" + +from decimal import ROUND_HALF_UP, Decimal + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from policyengine_core.parameters import Parameter +from policyengine_core.reforms import Reform + +from policyengine_uk import Simulation +from policyengine_uk import tax_benefit_system as tbs_module +from policyengine_uk.utils.dated_reforms import ( + calendar_year_bounds, + read_parameter_changes, + read_reform_dict, +) +from policyengine_uk.utils.scenario import Scenario + +MINIMUM_RATE = "gov.dwp.state_pension.triple_lock.minimum_rate" +OUTTURN = "gov.dwp.state_pension.triple_lock.outturn" +GROWTH = "gov.economic_assumptions.yoy_growth" +AVERAGE_EARNINGS = f"{GROWTH}.obr.average_earnings" +TRIPLE_LOCK = f"{GROWTH}.triple_lock" +LAGGED_CPI = f"{GROWTH}.obr.lagged_cpi" +INDICES = "gov.economic_assumptions.indices" +CPI_INDEX = f"{INDICES}.obr.consumer_price_index" +INPUTS = "gov.economic_assumptions.statutory_uprating_inputs" +AWE = f"{INPUTS}.awe_total_pay_may_july" +AWE_GAP = f"{INPUTS}.forecast_gap.awe_total_pay_may_july" +CPI_SEPTEMBER = f"{INPUTS}.cpi_september" +NEW_STATE_PENSION = "gov.dwp.state_pension.new_state_pension.amount" +BASIC_STATE_PENSION = "gov.dwp.state_pension.basic_state_pension.amount" +POVERTY_LINE = "household.poverty.absolute_poverty_threshold_bhc" +FINANCIAL_ASSETS = "household.wealth.financial_assets" +CORPORATE_WEALTH = "household.wealth.corporate_wealth" +PERSONAL_ALLOWANCE = "gov.hmrc.income_tax.allowances.personal_allowance.amount" +LSR_LEGACY = "gov.simulation.labor_supply_responses.income_elasticity" +LSR = "gov.simulation.labour_supply_responses.income_elasticity" +FAR = "2100-12-31" + +PROPERTY_SETTINGS = settings( + max_examples=10, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much], +) + +PENSIONER = { + "people": { + "pensioner": { + "age": {2025: 70}, + "state_pension_type": {2025: "NEW"}, + # Above the full rate, so the amount paid is the full rate. + "state_pension_reported": {2025: 50_000}, + } + }, + "benunits": {"benunit": {"members": ["pensioner"]}}, + "households": {"household": {"members": ["pensioner"]}}, +} + + +def value(simulation, path, instant): + return simulation.tax_benefit_system.parameters.get_child(path)(instant) + + +def weekly(simulation, path, year): + """A State Pension rate in force over fiscal year ``year``.""" + return float(value(simulation, path, f"{year}-04-30")) + + +def growth(simulation, path, year): + """An index's growth over calendar year ``year``.""" + return ( + value(simulation, path, f"{year}-01-01") + / value(simulation, path, f"{year - 1}-01-01") + - 1 + ) + + +def published(rate): + """A growth rate to 0.1 percentage points, halves away from zero.""" + return float(Decimal(repr(float(rate))).quantize(Decimal("0.001"), ROUND_HALF_UP)) + + +def via_reform_dict(change, situation=PENSIONER): + return Simulation(situation=situation, reform=change) + + +def via_parameter_changes(change, situation=PENSIONER): + return Simulation(situation=situation, scenario=Scenario(parameter_changes=change)) + + +def via_scenario_before_data_load(change, situation=PENSIONER): + return Simulation( + situation=situation, + scenario=Scenario(parameter_changes=change, applied_before_data_load=True), + ) + + +ROUTES = [via_reform_dict, via_parameter_changes, via_scenario_before_data_load] + + +@pytest.fixture(scope="module") +def baseline(): + return Simulation(situation=PENSIONER) + + +# 1. Inputs reach what is built from them. + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize( + "spec", + [0.1, {"2028": 0.1}, {f"2028-01-01.{FAR}": 0.1}], + ids=["scalar", "bare-year", "interval"], +) +def test_the_minimum_rate_moves_the_triple_lock_and_the_state_pension( + route, spec, baseline +): + reformed = route({MINIMUM_RATE: spec}) + assert value(reformed, TRIPLE_LOCK, "2028-01-01") == pytest.approx(0.1) + for path in (NEW_STATE_PENSION, BASIC_STATE_PENSION): + if isinstance(spec, dict): + # A change from 2028 leaves the April 2027 uprating alone; a + # scalar applies from 2023 (or 2000) and so raises it too. + assert weekly(reformed, path, 2027) == weekly(baseline, path, 2027) + assert weekly(reformed, path, 2028) == pytest.approx( + weekly(reformed, path, 2027) * 1.1, abs=0.006 + ) + pension = float(reformed.calculate("new_state_pension", 2028).sum()) + assert pension == pytest.approx( + weekly(reformed, NEW_STATE_PENSION, 2028) * 52, rel=1e-6 + ) + + +@PROPERTY_SETTINGS +@given( + low=st.integers(0, 150).map(lambda n: n / 1000), + high=st.integers(0, 150).map(lambda n: n / 1000), +) +def test_the_state_pension_is_non_decreasing_in_the_minimum_rate(low, high): + low, high = sorted((low, high)) + pensions = [ + weekly(via_reform_dict({MINIMUM_RATE: rate}), NEW_STATE_PENSION, 2028) + for rate in (low, high) + ] + assert pensions[0] <= pensions[1] + + +@PROPERTY_SETTINGS +@given( + rate=st.integers(-50, 200).map(lambda n: n / 1000 + 0.0003), + year=st.integers(2027, 2030), +) +def test_earnings_growth_sets_the_next_aprils_triple_lock_by_its_rule( + rate, year, baseline +): + reformed = via_reform_dict({AVERAGE_EARNINGS: {str(year): rate}}) + # The May to July input is calendar-year growth plus the OBR gap. + gap = value(baseline, AWE_GAP, f"{year}-07-01") or 0 + earnings = value(reformed, AWE, f"{year}-07-01") + assert earnings == pytest.approx(rate + gap, abs=1e-12) + cpi = value(reformed, CPI_SEPTEMBER, f"{year}-09-01") + expected = max( + value(reformed, MINIMUM_RATE, f"{year + 1}-04-30"), + published(rate + gap), + published(cpi), + ) + assert value(reformed, TRIPLE_LOCK, f"{year + 1}-01-01") == pytest.approx(expected) + # The year before is untouched. + assert value(reformed, TRIPLE_LOCK, f"{year}-01-01") == value( + baseline, TRIPLE_LOCK, f"{year}-01-01" + ) + assert weekly(reformed, NEW_STATE_PENSION, year + 1) == pytest.approx( + weekly(reformed, NEW_STATE_PENSION, year) * (1 + expected), abs=0.006 + ) + + +# 2. A year is one year on a calendar-year series. + + +CALENDAR_KEYS = [ + "2027", + 2027, + "2027-01-01", + "year:2027-01-01:1", + "2027-01-01.2027-12-31", +] + + +def observed(simulation): + """What processing built from 2027 earnings growth, and what it shows.""" + years = range(2025, 2032) + return ( + [value(simulation, AVERAGE_EARNINGS, f"{y}-01-01") for y in years], + [ + value(simulation, f"{INDICES}.obr.average_earnings", f"{y}-01-01") + for y in years + ], + [value(simulation, AWE, f"{y}-07-01") for y in years], + [value(simulation, TRIPLE_LOCK, f"{y}-01-01") for y in years], + [weekly(simulation, NEW_STATE_PENSION, y) for y in years], + ) + + +@pytest.mark.parametrize("route", [via_reform_dict, via_parameter_changes]) +def test_every_key_for_a_year_of_growth_gives_the_same_parameters(route): + results = [observed(route({AVERAGE_EARNINGS: {key: 0.2}})) for key in CALENDAR_KEYS] + assert all(result == results[0] for result in results) + + +def test_the_growth_shown_for_a_year_is_the_growth_used_for_it(baseline): + reformed = via_reform_dict({AVERAGE_EARNINGS: {"2027": 0.2}}) + assert value(reformed, AVERAGE_EARNINGS, "2027-01-01") == 0.2 + assert growth(reformed, f"{INDICES}.obr.average_earnings", 2027) == pytest.approx( + 0.2, abs=1e-4 + ) + assert growth(reformed, f"{INDICES}.obr.average_earnings", 2028) == pytest.approx( + growth(baseline, f"{INDICES}.obr.average_earnings", 2028), abs=1e-4 + ) + gap = value(baseline, AWE_GAP, "2027-07-01") + assert value(reformed, AWE, "2027-07-01") == pytest.approx(0.2 + gap) + assert value(reformed, AWE, "2028-07-01") == value(baseline, AWE, "2028-07-01") + assert value(reformed, TRIPLE_LOCK, "2028-01-01") == pytest.approx(0.2) + assert value(reformed, TRIPLE_LOCK, "2029-01-01") == value( + baseline, TRIPLE_LOCK, "2029-01-01" + ) + + +def statutory_growth_leaves(): + growth_node = tbs_module.statutory_parameters().get_child(GROWTH) + return sorted( + parameter.name + for parameter in growth_node.get_descendants() + if isinstance(parameter, Parameter) + and all(isinstance(v.value, (int, float)) for v in parameter.values_list) + ) + + +GROWTH_LEAVES = statutory_growth_leaves() + + +@PROPERTY_SETTINGS +@given( + path=st.sampled_from(GROWTH_LEAVES), + year=st.integers(2025, 2035), + rate=st.integers(-50, 300).map(lambda n: n / 1000), +) +def test_each_growth_rate_moves_its_index_in_the_year_it_is_shown( + path, year, rate, baseline +): + index = path.replace("yoy_growth", "indices") + reformed = via_reform_dict({path: {str(year): rate}}) + assert value(reformed, path, f"{year}-01-01") == pytest.approx(rate) + assert growth(reformed, index, year) == pytest.approx(rate, abs=2e-4) + for other in (year - 1, year + 1): + assert growth(reformed, index, other) == pytest.approx( + growth(baseline, index, other), abs=2e-4 + ) + + +def test_a_bare_year_on_the_household_tree_is_that_calendar_year(baseline): + reformed = via_reform_dict({FINANCIAL_ASSETS: {"2027": 1e13}}) + assert value(reformed, FINANCIAL_ASSETS, "2027-01-01") == 1e13 + assert value(reformed, FINANCIAL_ASSETS, "2028-01-01") == value( + baseline, FINANCIAL_ASSETS, "2028-01-01" + ) + # Corporate wealth is uprated by financial assets, in the same year. + ratio = 1e13 / value(baseline, FINANCIAL_ASSETS, "2027-01-01") + assert value(reformed, CORPORATE_WEALTH, "2027-01-01") == pytest.approx( + value(baseline, CORPORATE_WEALTH, "2027-01-01") * ratio, rel=1e-9 + ) + assert value(reformed, CORPORATE_WEALTH, "2028-01-01") == value( + baseline, CORPORATE_WEALTH, "2028-01-01" + ) + + +def test_calendar_year_series_are_the_growth_rates_indices_and_household_tree(): + reads = tbs_module.parameter_reads_calendar_years() + paths = tbs_module._leaf_paths(tbs_module.processed_parameters()) + for path in paths: + if path.startswith("baseline."): + continue + expected = path.startswith((f"{GROWTH}.", f"{INDICES}.", "household.")) + assert reads(path) is expected, path + assert not reads("gov.not.a.parameter") + + +# 3. Generated series take effect. + + +@pytest.mark.parametrize( + "change", + [ + {TRIPLE_LOCK: {"2028": 0.05}}, + {TRIPLE_LOCK: {"2028-01-01.2028-12-31": 0.05}}, + # The statutory override of the same rate (read at 30 April). + {OUTTURN: {"2028-04-06.2029-04-05": 0.05}}, + ], + ids=["generated-bare-year", "generated-interval", "statutory-outturn"], +) +@pytest.mark.parametrize("route", ROUTES) +def test_a_triple_lock_rate_set_by_reform_uprates_the_state_pension( + route, change, baseline +): + reformed = route(change) + years = range(2026, 2036) + assert value(reformed, TRIPLE_LOCK, "2028-01-01") == pytest.approx(0.05) + for year in years: + if year == 2028: + continue + assert value(reformed, TRIPLE_LOCK, f"{year}-01-01") == value( + baseline, TRIPLE_LOCK, f"{year}-01-01" + ) + pensions = [weekly(reformed, NEW_STATE_PENSION, year) for year in years] + reference = [ + weekly( + via_reform_dict({OUTTURN: {"2028-04-06.2029-04-05": 0.05}}), + NEW_STATE_PENSION, + y, + ) + for y in years + ] + assert pensions == reference + assert pensions[2] == pytest.approx( + weekly(baseline, NEW_STATE_PENSION, 2027) * 1.05, abs=0.006 + ) + + +def test_reform_from_dict_sets_a_generated_series(): + change = {TRIPLE_LOCK: {"2028-01-01.2028-12-31": 0.05}} + reformed = Simulation(situation=PENSIONER, reform=Reform.from_dict(change)) + assert weekly(reformed, NEW_STATE_PENSION, 2028) == weekly( + via_reform_dict(change), NEW_STATE_PENSION, 2028 + ) + + +def test_a_changed_index_uprates_the_parameters_indexed_by_it(baseline): + level = value(baseline, CPI_INDEX, "2029-01-01") * 1.1 + reformed = via_reform_dict({CPI_INDEX: {"2029": level}}) + for year, factor in ((2028, 1), (2029, 1.1), (2030, 1)): + assert value(reformed, POVERTY_LINE, f"{year}-01-01") == pytest.approx( + value(baseline, POVERTY_LINE, f"{year}-01-01") * factor, rel=1e-4 + ) + + +def test_a_lagged_series_moves_its_index(baseline): + reformed = via_reform_dict({LAGGED_CPI: {"2028": 0.1}}) + assert growth(reformed, f"{INDICES}.obr.lagged_cpi", 2028) == pytest.approx( + 0.1, abs=1e-4 + ) + assert growth(reformed, f"{INDICES}.obr.lagged_cpi", 2029) == pytest.approx( + growth(baseline, f"{INDICES}.obr.lagged_cpi", 2029), abs=1e-4 + ) + + +def test_generated_parameters_are_the_series_processing_builds(): + generates = tbs_module.generated_parameter_defines() + statutory = tbs_module.statutory_parameter_defines() + processed = tbs_module.processed_parameters() + for path in tbs_module._leaf_paths(processed): + if path.startswith("baseline.") or path.startswith( + "gov.simulation.labor_supply_responses." + ): + assert not generates(path), path + continue + built = path.startswith((f"{INDICES}.", f"{GROWTH}.")) and not statutory(path) + # The CGT projection index is added after processing, outside it. + if path == f"{INDICES}.capital_gains": + assert not generates(path) + continue + assert generates(path) is built, path + assert generates(TRIPLE_LOCK) + assert not generates(MINIMUM_RATE) + + +@pytest.mark.parametrize( + "route", + [ + via_reform_dict, + lambda change: Simulation(situation=PENSIONER, reform=Reform.from_dict(change)), + ], + ids=["reform-dict", "reform-from-dict"], +) +def test_a_labour_supply_alias_path_changes_the_parameter_it_copies(route): + reformed = route({LSR_LEGACY: {f"2025-01-01.{FAR}": 0.5}}) + assert value(reformed, LSR, "2028-01-01") == 0.5 + + +@pytest.mark.parametrize("route", [via_reform_dict, via_parameter_changes]) +def test_an_unknown_path_is_still_rejected(route): + with pytest.raises(ValueError, match="Could not find the parameter"): + route({"gov.not.a.parameter": {"2027": 1}}) + + +# 4. Ordinary reforms are unaffected. + + +def statutory_leaves(): + return tbs_module._leaf_paths(tbs_module.statutory_parameters()) + + +ORDINARY = [ + path + for path in statutory_leaves() + if path.startswith("gov.") + and not path.startswith(("gov.economic_assumptions.yoy_growth.",)) +] +KEYS = st.one_of( + st.integers(2016, 2040), + st.integers(2016, 2040).map(str), + st.integers(2016, 2040).map(lambda y: f"{y}-01-01"), + st.integers(2016, 2040).map(lambda y: f"{y}-04-06"), + st.integers(2016, 2040).map(lambda y: f"year:{y}-01-01:1"), + st.integers(2016, 2040).map(lambda y: f"{y}-04-06.{y + 1}-04-05"), +) + + +@settings(max_examples=200, deadline=None, derandomize=True) +@given(path=st.sampled_from(ORDINARY), key=KEYS) +def test_ordinary_parameters_read_every_key_as_before(path, key): + reads = tbs_module.parameter_reads_calendar_years() + defines = tbs_module.reform_parameter_defines() + change = {path: {key: 1}} + assert read_reform_dict(change, defines, calendar_years=reads) == read_reform_dict( + change, defines + ) + assert read_parameter_changes( + change, calendar_years=reads + ) == read_parameter_changes(change) + + +def test_an_ordinary_bare_year_is_still_the_fiscal_year(baseline): + reformed = via_reform_dict({PERSONAL_ALLOWANCE: {"2027": 20_000}}) + interval = via_reform_dict({PERSONAL_ALLOWANCE: {"2027-04-06.2028-04-05": 20_000}}) + for year in range(2025, 2032): + assert value(reformed, PERSONAL_ALLOWANCE, f"{year}-01-01") == value( + interval, PERSONAL_ALLOWANCE, f"{year}-01-01" + ) + assert value(reformed, NEW_STATE_PENSION, "2028-04-30") == value( + baseline, NEW_STATE_PENSION, "2028-04-30" + ) + + +def test_calendar_year_bounds(): + start, stop = calendar_year_bounds(2027) + assert (str(start), str(stop)) == ("2027-01-01", "2027-12-31") + + +# 6. Baselines stay current law. + + +@pytest.mark.parametrize( + "change", + [ + {MINIMUM_RATE: 0.1}, + {AVERAGE_EARNINGS: {"2027": 0.2}}, + {TRIPLE_LOCK: {"2028": 0.05}}, + ], + ids=["input", "calendar-year-series", "generated"], +) +def test_baselines_stay_current_law(change, baseline): + reformed = via_reform_dict(change) + for year in (2028, 2029): + assert weekly(reformed.baseline, NEW_STATE_PENSION, year) == weekly( + baseline, NEW_STATE_PENSION, year + ) + assert float( + reformed.tax_benefit_system.parameters.baseline.get_child( + NEW_STATE_PENSION + )(f"{year}-04-30") + ) == weekly(baseline, NEW_STATE_PENSION, year) + after = Simulation(situation=PENSIONER) + assert weekly(after, NEW_STATE_PENSION, 2028) == weekly( + baseline, NEW_STATE_PENSION, 2028 + ) From 4090c108ac4271588ec65821c2e25057f329cc74 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 02:35:59 -0400 Subject: [PATCH 3/6] Accept the generated changes in the never-settling projection test's stand-in Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_dated_reform_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policyengine_uk/tests/test_dated_reform_routes.py b/policyengine_uk/tests/test_dated_reform_routes.py index d4b35393d3..66d83917a8 100644 --- a/policyengine_uk/tests/test_dated_reform_routes.py +++ b/policyengine_uk/tests/test_dated_reform_routes.py @@ -877,7 +877,7 @@ def test_prior_law_that_never_settles_is_an_error(monkeypatch): calls = {"n": 0} - def unsettled(tree, unreformed=None, annualise=True): + def unsettled(tree, unreformed=None, annualise=True, generated=None): calls["n"] += 1 entry = ParameterAtInstant( UC_SINGLE_OLD, "2026-04-01", data={"value": float(calls["n"])} From e6c115f3c7da186313ddd12e782980b7826a521b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 03:17:18 -0400 Subject: [PATCH 4/6] Address review: project household parameters' prior law, reject a path set twice, name the right year in warnings Prior law's projected path was recorded for gov parameters only, so a change to an uprated household parameter past its last stated value froze the years around it (the poverty line 5.5% low in 2026 and 14.4% low in 2031 for a 2030 change). It is now recorded for the household tree too, which also fixes bare-year and interval keys there. A reform setting a labour supply parameter by both its path and its deprecated alias dropped one change silently; it is now an error. The single-day key warning names the calendar year on calendar-year series. Tests: an uprated household parameter on every key and route, the alias collision, Scenario.apply and a reprocessing modifier on a generated series, Reform.from_dict against an oracle. Docs and changelog corrected. Co-Authored-By: Claude Opus 5.5 --- .../reform-dict-process-time-params.fixed.md | 2 +- docs/book/usage/scenarios.md | 4 +- .../gov/hmrc/create_threshold_indexation.py | 16 ++-- .../labour_supply_responses/aliases.py | 21 +++++ policyengine_uk/simulation.py | 6 +- policyengine_uk/tax_benefit_system.py | 24 ++++-- .../tests/test_process_time_reforms.py | 78 +++++++++++++++---- policyengine_uk/utils/dated_reforms.py | 16 ++-- policyengine_uk/utils/scenario.py | 12 +-- 9 files changed, 130 insertions(+), 49 deletions(-) diff --git a/changelog.d/reform-dict-process-time-params.fixed.md b/changelog.d/reform-dict-process-time-params.fixed.md index 34985128fc..b1f7561989 100644 --- a/changelog.d/reform-dict-process-time-params.fixed.md +++ b/changelog.d/reform-dict-process-time-params.fixed.md @@ -1 +1 @@ -A bare-year reform key on the economic growth rates, the indices built from them and the household parameters now names that calendar year. It used to name the fiscal year from 6 April, which moved the next year: a 2027 earnings growth rate showed as 2027's but moved the 2028 index and the April 2029 triple lock. Reforms to series built during processing (the triple lock rate, the indices, the lagged series) now uprate what is built from them, and deprecated `labor_supply_responses` paths in reform dicts reach the parameters formulas read. +A bare-year reform key, or a key dated 1 January, on the economic growth rates, the indices built from them and the household parameters now names that calendar year. A bare year used to name the fiscal year from 6 April, which moved the next year: a 2027 earnings growth rate showed as 2027's but moved the 2028 index and the April 2029 triple lock. A 1 January key used to override that one day after processing, which nothing built from the parameter read. A change to an uprated household parameter past its last stated value now leaves the years around it on their projected path instead of freezing them. Reforms to series built during processing (the triple lock rate, the indices, the lagged series) now uprate what is built from them. Deprecated `labor_supply_responses` paths in reform dicts reach the parameters formulas read, and a reform that sets a parameter by both its path and its alias is an error instead of silently dropping one. diff --git a/docs/book/usage/scenarios.md b/docs/book/usage/scenarios.md index 74099afbdb..4aaa7a18e4 100644 --- a/docs/book/usage/scenarios.md +++ b/docs/book/usage/scenarios.md @@ -394,11 +394,11 @@ Each way of passing a reform reads period keys as it always has: | `"2027-04-06"` | that one day of the annualised value | that one day | from that day onwards | | a value with no period | from 1 January 2023 onwards | 2000 to 2099 | 2000 to 2099 | -A single-date key in a `reform=` dict overrides the annualised value on that day only. Formulas read annual values on 1 January, so `"2027-01-01"` sets fiscal year 2027 and `"2027-04-06"` changes nothing. The model logs a warning when a key can change nothing. Use `"start.stop"` keys for dated measures. +A single-date key in a `reform=` dict overrides the annualised value on that day only. Formulas read annual values on 1 January, so `"2027-01-01"` sets fiscal year 2027 and `"2027-04-06"` changes nothing. On a calendar-year series (below), a 1 January key names that calendar year instead. The model logs a warning when a key can change nothing. Use `"start.stop"` keys for dated measures. Some parameters hold one value per calendar year, dated 1 January, and are read there: the growth rates under `gov.economic_assumptions.yoy_growth`, the indices built from them under `gov.economic_assumptions.indices`, and the `household` parameters, which are not annualised. On these a bare year, or its 1 January, names the calendar year. So `{"gov.economic_assumptions.yoy_growth.obr.average_earnings": {"2027": 0.05}}` sets earnings growth in 2027: the 2027 earnings index, the May to July 2027 earnings input and the April 2028 triple lock all follow it. -Processing also builds some parameters from others: the triple lock rate (`gov.economic_assumptions.yoy_growth.triple_lock`), the lagged and private pension series, the September CPI uprating rate and the indices. A change to one is written as soon as it is built, so everything built from it follows. `{"gov.economic_assumptions.yoy_growth.triple_lock": {"2028": 0.05}}` uprates the State Pension by 5% in April 2028, exactly as `gov.dwp.state_pension.triple_lock.outturn` set for that April does. A change to an index uprates the parameters indexed by it. The deprecated `gov.simulation.labor_supply_responses` paths change the `labour_supply_responses` parameters they copy. +Processing also builds some parameters from others: the triple lock rate (`gov.economic_assumptions.yoy_growth.triple_lock`), the lagged and private pension series, the September CPI uprating rate and the indices. A change to one is written as soon as it is built, so everything built from it follows. `{"gov.economic_assumptions.yoy_growth.triple_lock": {"2028": 0.05}}` uprates the State Pension by 5% in April 2028, exactly as `gov.dwp.state_pension.triple_lock.outturn` set for that April does. A change to an index uprates whatever is uprated by it. The deprecated `gov.simulation.labor_supply_responses` paths change the `labour_supply_responses` parameters they copy. Changes applied after the data is loaded (the default) leave the data as built, so a changed economic assumption does not re-uprate incomes. Dated changes (`parameter_changes` and `"start.stop"` or bare-year keys) still come before the steps that read parameters while the simulation is built: the Universal Credit rebalancing awards and the structural reforms that parameters switch on. Single-day keys and a scenario's own simulation modifier run after those steps, as they always have. Set `applied_before_data_load=True` on a scenario to project the data with changed economic assumptions too. diff --git a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py index 0499a85c81..80043b724d 100644 --- a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py +++ b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py @@ -65,13 +65,15 @@ parameters carry no ``uprating`` metadata, so core uprating does not extend them. -The values are computed when the parameters are processed. A reform passed as -a dict edits the processed parameters, so changing one of these parameters -does not move the others. A scenario whose ``parameter_changes`` edit the raw -parameters, or a simulation modifier that resets and reprocesses them (as -``no_economic_assumptions`` does), rebuilds the values. So a changed CPI -forecast moves all seven, and a raw edit to the personal allowance or basic -rate limit after 2030-31 also moves the aligned NICs thresholds. +The values are computed when the parameters are processed. Every reform route +(a reform dict, ``parameter_changes``, ``Reform.from_dict``) writes its changes +onto the parameters before they are processed, as does a simulation modifier +that resets and reprocesses them (as ``no_economic_assumptions`` does), so the +values are rebuilt with the change: a changed CPI forecast moves all seven, +and a change to the personal allowance or basic rate limit after 2030-31 also +moves the aligned NICs thresholds (see ``policyengine_uk.utils.dated_reforms``). +``Scenario.apply`` on an already-built simulation is the exception: it edits +the processed parameters, so it does not move the others. """ from decimal import ROUND_CEILING, ROUND_HALF_UP, Decimal diff --git a/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py b/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py index b1c6ba523a..59c2b83b54 100644 --- a/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py +++ b/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py @@ -24,6 +24,27 @@ def canonicalize_lsr_parameter_path(path: str) -> str: return path +def canonicalize_parameter_changes(changes: dict) -> dict: + """Changes keyed by canonical parameter paths. + + A deprecated alias path names the parameter it copies, the one formulas + read. Two keys naming the same parameter (a path and its alias) would + leave one change unapplied, so they are an error. + """ + canonical = {} + given = {} + for path, value in changes.items(): + name = canonicalize_lsr_parameter_path(path) + if name in canonical: + raise ValueError( + f"The reform sets {name} twice, as {given[name]} and as {path}. " + f"Set it once, as {name}." + ) + canonical[name] = value + given[name] = path + return canonical + + def add_lsr_deprecation_aliases(parameters: ParameterNode) -> ParameterNode: simulation = parameters.gov.simulation canonical = simulation.labour_supply_responses diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index da0b59b28a..494533193c 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -61,7 +61,7 @@ ) from policyengine_uk.reforms import create_structural_reforms_from_parameters from policyengine_uk.parameters.gov.simulation.labour_supply_responses.aliases import ( - canonicalize_lsr_parameter_path, + canonicalize_parameter_changes, ) from .tax_benefit_system import ( @@ -73,9 +73,7 @@ def _canonical_parameter_paths(changes: dict) -> dict: """Changes keyed by canonical parameter paths (labour supply aliases).""" - return { - canonicalize_lsr_parameter_path(path): value for path, value in changes.items() - } + return canonicalize_parameter_changes(changes) from microdf import MicroDataFrame diff --git a/policyengine_uk/tax_benefit_system.py b/policyengine_uk/tax_benefit_system.py index abc6c5c686..3ae4bff7fc 100644 --- a/policyengine_uk/tax_benefit_system.py +++ b/policyengine_uk/tax_benefit_system.py @@ -85,9 +85,9 @@ # parameter's statutory values at their legal dates. Reforms are written # onto a copy of this tree. _statutory_parameters_cache = None -# Each gov parameter's timeline in the cached processed tree after -# projection (uprating, indexation and the other generated series) and -# before fiscal-year annualisation, by name. +# Each parameter's timeline in the cached processed tree after projection +# (uprating, indexation and the other generated series) and, for gov +# parameters, before fiscal-year annualisation, by name. _projected_timelines_cache = None @@ -151,8 +151,9 @@ def process_parameter_tree( - parameter uprating and backdating - conversion to fiscal year parameters - Returns the tree and each gov parameter's timeline just before the - conversion to fiscal years. + Returns the tree and each parameter's timeline after projection (for gov + parameters, just before the conversion to fiscal years), leaving out the + ``baseline`` copy. ``parameters.baseline`` is the copy of the parameters without reforms that ``baseline_vat`` and the budget adjustments compare against, taken @@ -179,7 +180,14 @@ def process_parameter_tree( parameters = propagate_parameter_metadata(parameters) parameters = uprate_parameters(parameters) parameters = backdate_parameters(parameters, BACKDATED_FROM) + # Prior law's projected path for every parameter a reform can change: the + # gov tree, before annualisation, and the trees outside it (household), + # which are not annualised. ``baseline`` is a copy, whose parameters + # keep the names of the ones it copies. projected = snapshot_timelines(parameters.gov) + for key, child in parameters.children.items(): + if key not in ("gov", "baseline") and isinstance(child, ParameterNode): + projected.update(snapshot_timelines(child)) if not annualise: return parameters, projected parameters.gov = convert_to_fiscal_year_parameters(parameters.gov) @@ -235,7 +243,7 @@ def process_parameters(self) -> None: """Process and transform parameters with UK-specific adjustments. See ``process_parameter_tree``, which this runs on the system's - parameters. ``projected_timelines`` keeps each gov parameter's + parameters. ``projected_timelines`` keeps each parameter's timeline from before the conversion to fiscal years. The system's dated reform changes (``dated_assignments``) are written @@ -384,7 +392,7 @@ def add_legacy_overrides(self, overrides: List[LegacyOverride]) -> None: @property def projected_timelines(self) -> Dict[str, Timeline]: - """Each gov parameter's timeline before fiscal-year annualisation. + """Each parameter's projected timeline (gov: before annualisation). A copy: the timelines of an unreformed system, and of a rebuild taken from the memo, are shared with later systems. @@ -598,7 +606,7 @@ def reads(path: str) -> bool: def projected_timelines() -> Dict[str, Timeline]: - """Current law's projected timeline for each gov parameter, by name. + """Current law's projected timeline for each parameter, by name. Shared: do not change them. ``CountryTaxBenefitSystem.projected_timelines`` returns a copy. diff --git a/policyengine_uk/tests/test_process_time_reforms.py b/policyengine_uk/tests/test_process_time_reforms.py index ab6d24107d..d508ae0b01 100644 --- a/policyengine_uk/tests/test_process_time_reforms.py +++ b/policyengine_uk/tests/test_process_time_reforms.py @@ -39,7 +39,7 @@ from policyengine_uk import Simulation from policyengine_uk import tax_benefit_system as tbs_module from policyengine_uk.utils.dated_reforms import ( - calendar_year_bounds, + ReformPeriodWarning, read_parameter_changes, read_reform_dict, ) @@ -356,11 +356,11 @@ def test_a_triple_lock_rate_set_by_reform_uprates_the_state_pension( ) -def test_reform_from_dict_sets_a_generated_series(): +def test_reform_from_dict_sets_a_generated_series(baseline): change = {TRIPLE_LOCK: {"2028-01-01.2028-12-31": 0.05}} reformed = Simulation(situation=PENSIONER, reform=Reform.from_dict(change)) - assert weekly(reformed, NEW_STATE_PENSION, 2028) == weekly( - via_reform_dict(change), NEW_STATE_PENSION, 2028 + assert weekly(reformed, NEW_STATE_PENSION, 2028) == pytest.approx( + weekly(baseline, NEW_STATE_PENSION, 2027) * 1.05, abs=0.006 ) @@ -394,13 +394,11 @@ def test_generated_parameters_are_the_series_processing_builds(): assert not generates(path), path continue built = path.startswith((f"{INDICES}.", f"{GROWTH}.")) and not statutory(path) - # The CGT projection index is added after processing, outside it. - if path == f"{INDICES}.capital_gains": - assert not generates(path) - continue assert generates(path) is built, path assert generates(TRIPLE_LOCK) assert not generates(MINIMUM_RATE) + # The CGT projection index is added after processing, outside it. + assert not generates(f"{INDICES}.capital_gains") @pytest.mark.parametrize( @@ -422,6 +420,65 @@ def test_an_unknown_path_is_still_rejected(route): route({"gov.not.a.parameter": {"2027": 1}}) +@pytest.mark.parametrize("route", [via_reform_dict, via_parameter_changes]) +@pytest.mark.parametrize("key", ["2030", "2030-01-01", "2030-01-01.2030-12-31"]) +def test_a_year_of_an_uprated_household_parameter_leaves_the_others_on_prior_law( + route, key, baseline +): + # The poverty line's stated values end before 2030; uprating projects it. + reformed = route({POVERTY_LINE: {key: 500}}) + for year in range(2025, 2036): + expected = ( + 500 if year == 2030 else value(baseline, POVERTY_LINE, f"{year}-01-01") + ) + assert value(reformed, POVERTY_LINE, f"{year}-01-01") == pytest.approx(expected) + + +def test_a_reprocessing_modifier_keeps_a_generated_series_change(baseline): + from policyengine_uk.scenarios import no_economic_assumptions + + scenario = ( + Scenario.from_reform({TRIPLE_LOCK: {"2028": 0.05}}) + no_economic_assumptions + ) + reformed = Simulation(situation=PENSIONER, scenario=scenario) + assert value(reformed, TRIPLE_LOCK, "2028-01-01") == pytest.approx(0.05) + assert weekly(reformed, NEW_STATE_PENSION, 2028) == pytest.approx( + weekly(reformed, NEW_STATE_PENSION, 2027) * 1.05, abs=0.006 + ) + + +def test_scenario_apply_on_a_built_simulation_keeps_its_legacy_reading(baseline): + # Scenario.apply edits the annualised parameters and never rebuilds them. + simulation = Simulation(situation=PENSIONER) + Scenario.from_reform({TRIPLE_LOCK: {"2028": 0.05}}).apply(simulation) + assert value(simulation, TRIPLE_LOCK, "2028-01-01") == pytest.approx(0.05) + assert weekly(simulation, NEW_STATE_PENSION, 2028) == weekly( + baseline, NEW_STATE_PENSION, 2028 + ) + + +@pytest.mark.parametrize("route", [via_reform_dict, via_parameter_changes]) +def test_a_path_and_its_alias_in_one_reform_are_an_error(route): + with pytest.raises(ValueError, match="twice"): + route({LSR: {f"2030-01-01.{FAR}": 0.2}, LSR_LEGACY: {f"2025-01-01.{FAR}": 0.1}}) + + +@pytest.mark.parametrize( + "path, kind", + [(AVERAGE_EARNINGS, "calendar"), (PERSONAL_ALLOWANCE, "fiscal")], +) +def test_a_single_day_key_warning_names_the_years_a_bare_year_covers(path, kind): + with pytest.warns( + ReformPeriodWarning, match=f"for one {kind} year use the bare year" + ): + read_reform_dict( + {path: {"2027-04-06": 1}}, + tbs_module.reform_parameter_defines(), + warn=True, + calendar_years=tbs_module.parameter_reads_calendar_years(), + ) + + # 4. Ordinary reforms are unaffected. @@ -471,11 +528,6 @@ def test_an_ordinary_bare_year_is_still_the_fiscal_year(baseline): ) -def test_calendar_year_bounds(): - start, stop = calendar_year_bounds(2027) - assert (str(start), str(stop)) == ("2027-01-01", "2027-12-31") - - # 6. Baselines stay current law. diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index 85761e17c9..ed18a858b4 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -43,7 +43,8 @@ How each route reads its period keys ------------------------------------ -The routes keep the readings they documented before: +The routes keep the readings they documented before, except on the +calendar-year series (starred, and below): ===================== ======================== ========================= ========================== Key reform dict ``parameter_changes`` ``Reform.from_dict`` @@ -90,7 +91,7 @@ under ``gov.economic_assumptions.indices``. A change to one is written onto its timeline just after the step that generates it, so the steps after it, and uprating, use it: a triple lock rate set for April 2028 uprates the -State Pension, and a changed index uprates the parameters indexed by it. +State Pension, and a changed index uprates whatever is uprated by it. Each route reads its keys as for the statutory parameters. The deprecated labour supply aliases (``gov.simulation.labor_supply_responses.*``) are copies of ``gov.simulation.labour_supply_responses.*``; a change to one is @@ -321,7 +322,11 @@ def read_reform_dict( start, stop = year elif is_sub_year(text): if warn: - _warn_sub_year_key(path, text) + _warn_sub_year_key( + path, + text, + calendar_years is not None and calendar_years(path), + ) legacy.append(LegacyOverride(path, text, value)) continue else: @@ -349,15 +354,16 @@ def _warn(message: str) -> None: warnings.warn(message, ReformPeriodWarning, stacklevel=4) -def _warn_sub_year_key(path: str, key: str) -> None: +def _warn_sub_year_key(path: str, key: str, calendar: bool = False) -> None: parsed = period(key) if parsed.unit == "day" and str(parsed.start)[5:] == "01-01": return + kind = "calendar" if calendar else "fiscal" _warn( f"Reform key '{key}' for {path} overrides the annualised parameter on " f"that {parsed.unit} only, and formulas read annual parameters at " f"1 January, so it may change nothing. For a change from that date " - f"use '{parsed.start}.{FLAT_STOP}'; for one fiscal year use the bare " + f"use '{parsed.start}.{FLAT_STOP}'; for one {kind} year use the bare " "year." ) diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index 6906021304..96ca131b47 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -52,18 +52,12 @@ def read( def _canonical_paths(changes: Dict[str, Any]) -> Dict[str, Any]: - """Changes keyed by canonical parameter paths. - - A deprecated labour supply alias path names the parameter it copies, - which is the one formulas read. - """ + """Changes keyed by canonical parameter paths (labour supply aliases).""" from policyengine_uk.parameters.gov.simulation.labour_supply_responses.aliases import ( - canonicalize_lsr_parameter_path, + canonicalize_parameter_changes, ) - return { - canonicalize_lsr_parameter_path(path): value for path, value in changes.items() - } + return canonicalize_parameter_changes(changes) def _is_reform_from_dict(reform: Type[Reform]) -> bool: From f9aa623915cb82de240e19936a20741555edefa9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:16:25 -0400 Subject: [PATCH 5/6] Address review round 2: resume before a series starts with its first value, merge a path with its alias A change ending before a series' first stated value (before 2015, where nothing is backdated) resumed with nothing, so building the index from a growth rate keyed two years early hit 1 + None; it now resumes with the first stated value, as backdating gives, and a stated null stays null. Two scenarios spelling a labour supply path differently now merge their changes in order instead of raising. Doc wording: single-day keys and the threshold indexation note name their exceptions. Co-Authored-By: Claude Opus 5.5 --- docs/book/usage/scenarios.md | 2 +- .../gov/hmrc/create_threshold_indexation.py | 5 ++- .../labour_supply_responses/aliases.py | 20 ++++----- .../tests/test_process_time_reforms.py | 42 +++++++++++++++++-- policyengine_uk/utils/dated_reforms.py | 13 +++++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/docs/book/usage/scenarios.md b/docs/book/usage/scenarios.md index 4aaa7a18e4..e8ef4a6d05 100644 --- a/docs/book/usage/scenarios.md +++ b/docs/book/usage/scenarios.md @@ -400,7 +400,7 @@ Some parameters hold one value per calendar year, dated 1 January, and are read Processing also builds some parameters from others: the triple lock rate (`gov.economic_assumptions.yoy_growth.triple_lock`), the lagged and private pension series, the September CPI uprating rate and the indices. A change to one is written as soon as it is built, so everything built from it follows. `{"gov.economic_assumptions.yoy_growth.triple_lock": {"2028": 0.05}}` uprates the State Pension by 5% in April 2028, exactly as `gov.dwp.state_pension.triple_lock.outturn` set for that April does. A change to an index uprates whatever is uprated by it. The deprecated `gov.simulation.labor_supply_responses` paths change the `labour_supply_responses` parameters they copy. -Changes applied after the data is loaded (the default) leave the data as built, so a changed economic assumption does not re-uprate incomes. Dated changes (`parameter_changes` and `"start.stop"` or bare-year keys) still come before the steps that read parameters while the simulation is built: the Universal Credit rebalancing awards and the structural reforms that parameters switch on. Single-day keys and a scenario's own simulation modifier run after those steps, as they always have. Set `applied_before_data_load=True` on a scenario to project the data with changed economic assumptions too. +Changes applied after the data is loaded (the default) leave the data as built, so a changed economic assumption does not re-uprate incomes. Dated changes (`parameter_changes` and `"start.stop"` or bare-year keys) still come before the steps that read parameters while the simulation is built: the Universal Credit rebalancing awards and the structural reforms that parameters switch on. Single-day keys (other than a 1 January key on a calendar-year series, which is a dated change) and a scenario's own simulation modifier run after those steps, as they always have. Set `applied_before_data_load=True` on a scenario to project the data with changed economic assumptions too. All of this applies when the scenario or reform is passed to `Simulation(...)`. `Scenario.apply(simulation)` on a simulation that is already built edits its annualised values instead, as it always has: a `"2027-04-06.2028-04-05"` key there overrides those dates of the fiscal-year values, so it does not set fiscal year 2027, and the model logs a warning saying so. Pass the scenario when constructing the simulation for dated measures. diff --git a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py index 80043b724d..bc5e77ed2d 100644 --- a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py +++ b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py @@ -72,8 +72,9 @@ values are rebuilt with the change: a changed CPI forecast moves all seven, and a change to the personal allowance or basic rate limit after 2030-31 also moves the aligned NICs thresholds (see ``policyengine_uk.utils.dated_reforms``). -``Scenario.apply`` on an already-built simulation is the exception: it edits -the processed parameters, so it does not move the others. +The exceptions edit the processed parameters, so they do not move the others: +``Scenario.apply`` on an already-built simulation, and a reform-dict key naming +a single day or month. """ from decimal import ROUND_CEILING, ROUND_HALF_UP, Decimal diff --git a/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py b/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py index 59c2b83b54..d88f3a7616 100644 --- a/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py +++ b/policyengine_uk/parameters/gov/simulation/labour_supply_responses/aliases.py @@ -28,20 +28,20 @@ def canonicalize_parameter_changes(changes: dict) -> dict: """Changes keyed by canonical parameter paths. A deprecated alias path names the parameter it copies, the one formulas - read. Two keys naming the same parameter (a path and its alias) would - leave one change unapplied, so they are an error. + read. Where a path and its alias both appear (two scenarios spelling it + differently, say), their changes are merged in order, as two changes to + one path are: dated keys from both are kept, a later key for the same + period wins, and a value with no period replaces what came before. """ canonical = {} - given = {} for path, value in changes.items(): name = canonicalize_lsr_parameter_path(path) - if name in canonical: - raise ValueError( - f"The reform sets {name} twice, as {given[name]} and as {path}. " - f"Set it once, as {name}." - ) - canonical[name] = value - given[name] = path + if name not in canonical: + canonical[name] = value + elif isinstance(canonical[name], dict) and isinstance(value, dict): + canonical[name] = {**canonical[name], **value} + else: + canonical[name] = value return canonical diff --git a/policyengine_uk/tests/test_process_time_reforms.py b/policyengine_uk/tests/test_process_time_reforms.py index d508ae0b01..dbbf617ecf 100644 --- a/policyengine_uk/tests/test_process_time_reforms.py +++ b/policyengine_uk/tests/test_process_time_reforms.py @@ -458,9 +458,45 @@ def test_scenario_apply_on_a_built_simulation_keeps_its_legacy_reading(baseline) @pytest.mark.parametrize("route", [via_reform_dict, via_parameter_changes]) -def test_a_path_and_its_alias_in_one_reform_are_an_error(route): - with pytest.raises(ValueError, match="twice"): - route({LSR: {f"2030-01-01.{FAR}": 0.2}, LSR_LEGACY: {f"2025-01-01.{FAR}": 0.1}}) +def test_a_path_and_its_alias_in_one_reform_are_merged_in_order(route): + reformed = route( + {LSR: {f"2030-01-01.{FAR}": 0.2}, LSR_LEGACY: {"2026-01-01.2027-12-31": 0.1}} + ) + for year, expected in ((2025, 0), (2026, 0.1), (2028, 0), (2030, 0.2)): + assert value(reformed, LSR, f"{year}-01-01") == pytest.approx(expected) + + +def test_scenarios_spelling_a_path_differently_combine(): + combined = Scenario(parameter_changes={LSR: {f"2030-01-01.{FAR}": 0.2}}) + Scenario( + parameter_changes={LSR_LEGACY: {"2026-01-01.2027-12-31": 0.1}} + ) + reformed = Simulation(situation=PENSIONER, scenario=combined) + assert value(reformed, LSR, "2026-01-01") == pytest.approx(0.1) + assert value(reformed, LSR, "2030-01-01") == pytest.approx(0.2) + + +@pytest.mark.parametrize("year", [2005, 2006, 2007, 2008]) +@pytest.mark.parametrize("path", [f"{GROWTH}.obr.rpi", TRIPLE_LOCK]) +def test_a_change_before_a_series_starts_resumes_with_its_first_value( + path, year, baseline +): + # Before 2015 nothing is backdated, so a change ending before a series' + # first stated value resumes with that value rather than with nothing. + reformed = via_reform_dict({path: {str(year): 0.1}}) + assert value(reformed, path, f"{year}-01-01") == pytest.approx(0.1) + for later in range(2016, 2030): + assert value(reformed, path, f"{later}-01-01") == value( + baseline, path, f"{later}-01-01" + ) + + +def test_a_stated_null_resumes_as_null(baseline): + # The triple lock override is null outside the years it overrides. + reformed = via_reform_dict({OUTTURN: {"2028-04-06.2029-04-05": 0.05}}) + assert value(reformed, OUTTURN, "2029-04-30") is None + assert value(reformed, TRIPLE_LOCK, "2029-01-01") == value( + baseline, TRIPLE_LOCK, "2029-01-01" + ) @pytest.mark.parametrize( diff --git a/policyengine_uk/utils/dated_reforms.py b/policyengine_uk/utils/dated_reforms.py index ed18a858b4..4b4e97c3c9 100644 --- a/policyengine_uk/utils/dated_reforms.py +++ b/policyengine_uk/utils/dated_reforms.py @@ -534,7 +534,9 @@ def write_assignment( value and the change, or after a bounded change, would lose their projection. Then, as ``Parameter.update`` does, entries within the change are replaced by its value and the value in force at ``stop`` is - restored on the next day unless an entry already starts then. + restored on the next day unless an entry already starts then. A change + that ends before the first stated value resumes with that value, as + processing would backdate it. """ name = parameter.name start_str = str(assignment.start) @@ -563,7 +565,14 @@ def write_assignment( earlier = [entry for entry in old if entry.instant_str < start_str] resumed: Timeline = [] if not later or later[-1].instant_str != next_str: - resumed.append(_entry(name, next_str, _value_in_force(old, stop_str))) + in_force = [entry for entry in old if entry.instant_str <= stop_str] + # Before its first stated value a parameter takes that value, as + # processing backdates it; resuming with nothing would leave the + # years up to that value empty. A stated null stays null. + resumed_value = ( + in_force[0].value if in_force else later[-1].value if later else None + ) + resumed.append(_entry(name, next_str, resumed_value)) values = later + resumed + [_entry(name, start_str, assignment.value)] + earlier values.sort(key=lambda entry: entry.instant_str, reverse=True) parameter.values_list = values From 461f6168b9eb838106a48ab682d7a73a8d7e4801 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:31:56 -0400 Subject: [PATCH 6/6] Address review round 3: canonicalise each Scenario sum operand before merging Scenario.__add__ merged parameter changes by their literal paths, and canonicalisation came later, so a scalar given by a labour supply alias did not replace an earlier change given by the canonical path, and a later dict then merged the replaced change back in. Each operand is now keyed by canonical paths before the merge, so a sum spelling a parameter two ways gives what the same sum spelt one way gives. The threshold indexation note and the changelog name the 1 January and merge rules. Co-Authored-By: Claude Opus 5.5 --- .../reform-dict-process-time-params.fixed.md | 2 +- .../gov/hmrc/create_threshold_indexation.py | 3 +- .../tests/test_process_time_reforms.py | 34 +++++++++++++++++++ policyengine_uk/utils/scenario.py | 8 +++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/changelog.d/reform-dict-process-time-params.fixed.md b/changelog.d/reform-dict-process-time-params.fixed.md index b1f7561989..8d19dff703 100644 --- a/changelog.d/reform-dict-process-time-params.fixed.md +++ b/changelog.d/reform-dict-process-time-params.fixed.md @@ -1 +1 @@ -A bare-year reform key, or a key dated 1 January, on the economic growth rates, the indices built from them and the household parameters now names that calendar year. A bare year used to name the fiscal year from 6 April, which moved the next year: a 2027 earnings growth rate showed as 2027's but moved the 2028 index and the April 2029 triple lock. A 1 January key used to override that one day after processing, which nothing built from the parameter read. A change to an uprated household parameter past its last stated value now leaves the years around it on their projected path instead of freezing them. Reforms to series built during processing (the triple lock rate, the indices, the lagged series) now uprate what is built from them. Deprecated `labor_supply_responses` paths in reform dicts reach the parameters formulas read, and a reform that sets a parameter by both its path and its alias is an error instead of silently dropping one. +A bare-year reform key, or a key dated 1 January, on the economic growth rates, the indices built from them and the household parameters now names that calendar year. A bare year used to name the fiscal year from 6 April, which moved the next year: a 2027 earnings growth rate showed as 2027's but moved the 2028 index and the April 2029 triple lock. A 1 January key used to override that one day after processing, which nothing built from the parameter read. A change to an uprated household parameter past its last stated value now leaves the years around it on their projected path instead of freezing them. Reforms to series built during processing (the triple lock rate, the indices, the lagged series) now uprate what is built from them. Deprecated `labor_supply_responses` paths in reform dicts reach the parameters formulas read, and changes to a parameter given by both its path and its alias are merged in order instead of one being silently dropped. diff --git a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py index bc5e77ed2d..c2b57a543a 100644 --- a/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py +++ b/policyengine_uk/parameters/gov/hmrc/create_threshold_indexation.py @@ -74,7 +74,8 @@ moves the aligned NICs thresholds (see ``policyengine_uk.utils.dated_reforms``). The exceptions edit the processed parameters, so they do not move the others: ``Scenario.apply`` on an already-built simulation, and a reform-dict key naming -a single day or month. +a single day or month (other than 1 January on a calendar-year series, such as +a CPI growth rate, which is a dated change and so does move them). """ from decimal import ROUND_CEILING, ROUND_HALF_UP, Decimal diff --git a/policyengine_uk/tests/test_process_time_reforms.py b/policyengine_uk/tests/test_process_time_reforms.py index dbbf617ecf..ec16289e45 100644 --- a/policyengine_uk/tests/test_process_time_reforms.py +++ b/policyengine_uk/tests/test_process_time_reforms.py @@ -475,6 +475,40 @@ def test_scenarios_spelling_a_path_differently_combine(): assert value(reformed, LSR, "2030-01-01") == pytest.approx(0.2) +WIDE = f"2025-01-01.{FAR}" +NARROW = "2026-01-01.2027-12-31" + + +@pytest.mark.parametrize( + "stages", + [ + [{WIDE: 0.1}, 0.2, {NARROW: 0.3}], + [{WIDE: 0.1}, {WIDE: 0.2}, {WIDE: 0.3}], + [0.1, {NARROW: 0.2}, 0.3], + [{NARROW: 0.1}, {WIDE: 0.2}, {NARROW: 0.3}], + ], + ids=["dict-scalar-dict", "same-period", "scalar-dict-scalar", "narrow-wide-narrow"], +) +@pytest.mark.parametrize( + "spellings", [(LSR, LSR_LEGACY, LSR), (LSR_LEGACY, LSR, LSR_LEGACY)] +) +def test_sums_of_scenarios_spelling_a_path_differently_match_one_spelling( + stages, spellings +): + def summed(paths): + scenarios = [ + Scenario(parameter_changes={path: stage}) + for path, stage in zip(paths, stages) + ] + return Simulation( + situation=PENSIONER, scenario=scenarios[0] + scenarios[1] + scenarios[2] + ) + + mixed, single = summed(spellings), summed((LSR, LSR, LSR)) + for year in (2024, 2025, 2026, 2028, 2031): + assert value(mixed, LSR, f"{year}-01-01") == value(single, LSR, f"{year}-01-01") + + @pytest.mark.parametrize("year", [2005, 2006, 2007, 2008]) @pytest.mark.parametrize("path", [f"{GROWTH}.obr.rpi", TRIPLE_LOCK]) def test_a_change_before_a_series_starts_resumes_with_its_first_value( diff --git a/policyengine_uk/utils/scenario.py b/policyengine_uk/utils/scenario.py index 96ca131b47..df3e02c162 100644 --- a/policyengine_uk/utils/scenario.py +++ b/policyengine_uk/utils/scenario.py @@ -172,14 +172,16 @@ def __add__(self, other: "Scenario") -> "Scenario": Returns: A new Scenario with merged parameter changes and combined modifiers """ - # Merge parameter changes (other's changes take precedence in conflicts) + # Merge parameter changes (other's changes take precedence in conflicts), + # keyed by canonical paths, so a parameter spelled two ways merges as + # one. merged_params = {} if self.parameter_changes: - merged_params.update(self.parameter_changes) + merged_params.update(_canonical_paths(self.parameter_changes)) if other.parameter_changes: - for key, value in other.parameter_changes.items(): + for key, value in _canonical_paths(other.parameter_changes).items(): if ( key in merged_params and isinstance(merged_params[key], dict)